diff --git "a/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/logs/debug-internal.log" "b/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/logs/debug-internal.log" @@ -0,0 +1,42138 @@ +2024-03-13 11:28:04,768 INFO StreamThr :877043 [internal.py:wandb_internal():86] W&B internal server running at pid: 877043, started at: 2024-03-13 11:28:04.766446 +2024-03-13 11:28:04,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status +2024-03-13 11:28:04,781 INFO WriterThread:877043 [datastore.py:open_for_write():87] open: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/run-pairvt6d.wandb +2024-03-13 11:28:04,783 DEBUG SenderThread:877043 [sender.py:send():382] send: header +2024-03-13 11:28:04,805 DEBUG SenderThread:877043 [sender.py:send():382] send: run +2024-03-13 11:28:05,318 INFO SenderThread:877043 [dir_watcher.py:__init__():211] watching files in: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files +2024-03-13 11:28:05,318 INFO SenderThread:877043 [sender.py:_start_run_threads():1136] run started: pairvt6d with start time 1710329284.76839 +2024-03-13 11:28:05,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: check_version +2024-03-13 11:28:05,329 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: check_version +2024-03-13 11:28:05,386 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: run_start +2024-03-13 11:28:05,396 DEBUG HandlerThread:877043 [system_info.py:__init__():27] System info init +2024-03-13 11:28:05,396 DEBUG HandlerThread:877043 [system_info.py:__init__():42] System info init done +2024-03-13 11:28:05,396 INFO HandlerThread:877043 [system_monitor.py:start():194] Starting system monitor +2024-03-13 11:28:05,396 INFO SystemMonitor:877043 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-13 11:28:05,396 INFO HandlerThread:877043 [system_monitor.py:probe():214] Collecting system info +2024-03-13 11:28:05,397 INFO SystemMonitor:877043 [interfaces.py:start():190] Started cpu monitoring +2024-03-13 11:28:05,397 INFO SystemMonitor:877043 [interfaces.py:start():190] Started disk monitoring +2024-03-13 11:28:05,397 INFO SystemMonitor:877043 [interfaces.py:start():190] Started gpu monitoring +2024-03-13 11:28:05,398 INFO SystemMonitor:877043 [interfaces.py:start():190] Started memory monitoring +2024-03-13 11:28:05,398 INFO SystemMonitor:877043 [interfaces.py:start():190] Started network monitoring +2024-03-13 11:28:05,468 DEBUG HandlerThread:877043 [system_info.py:probe():151] Probing system +2024-03-13 11:28:05,473 DEBUG HandlerThread:877043 [system_info.py:_probe_git():136] Probing git +2024-03-13 11:28:05,492 DEBUG HandlerThread:877043 [system_info.py:_probe_git():144] Probing git done +2024-03-13 11:28:05,492 DEBUG HandlerThread:877043 [system_info.py:probe():199] Probing system done +2024-03-13 11:28:05,492 DEBUG HandlerThread:877043 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-13T11:28:05.468416', 'startedAt': '2024-03-13T11:28:04.720032', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--stage', 'finetune', '--pretrained_checkpoint', '/home/lilei/prismatic-vlms/runs/llava-qformer/stage1_qformer2_128/checkpoints/latest-checkpoint.pt', '--model.model_id', 'one-stage+7b_qformer2_128', '--run_id', 's1_s2_qformer2_128', '--model.arch_specifier', 'qformer2_128'), 'state': 'running', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms.git', 'commit': 'e87c4d04915bf8fad0d90f2ae79454df778ee2d3'}, 'email': 'tobiaslee@qq.com', 'root': '/home/lilei/prismatic-vlms', 'host': 'compute-permanent-node-262', 'username': 'lilei', 'executable': '/home/lilei/anaconda3/envs/pvlm/bin/python', 'cpu_count': 128, 'cpu_count_logical': 255, 'cpu_freq': {'current': 2568.534541176471, 'min': 1500.0, 'max': 2550.0}, 'cpu_freq_per_core': [{'current': 3249.233, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.435, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3246.952, 'min': 1500.0, 'max': 2550.0}, {'current': 3166.559, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.518, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.637, 'min': 1500.0, 'max': 2550.0}, {'current': 2785.614, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.6, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.337, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.231, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}], 'disk': {'/': {'total': 242.1086883544922, 'used': 86.51881408691406}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.6942749023438}} +2024-03-13 11:28:05,493 INFO HandlerThread:877043 [system_monitor.py:probe():224] Finished collecting system info +2024-03-13 11:28:05,493 INFO HandlerThread:877043 [system_monitor.py:probe():227] Publishing system info +2024-03-13 11:28:05,493 DEBUG HandlerThread:877043 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-13 11:28:06,320 INFO Thread-12 :877043 [dir_watcher.py:_on_file_created():271] file/dir created: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/conda-environment.yaml +2024-03-13 11:28:10,055 DEBUG HandlerThread:877043 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-13 11:28:10,064 INFO HandlerThread:877043 [system_monitor.py:probe():229] Finished publishing system info +2024-03-13 11:28:10,070 DEBUG SenderThread:877043 [sender.py:send():382] send: files +2024-03-13 11:28:10,070 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-13 11:28:10,071 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:10,082 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: python_packages +2024-03-13 11:28:10,083 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: python_packages +2024-03-13 11:28:10,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:28:10,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:28:10,086 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:28:10,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/conda-environment.yaml +2024-03-13 11:28:10,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_created():271] file/dir created: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/requirements.txt +2024-03-13 11:28:10,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_created():271] file/dir created: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-metadata.json +2024-03-13 11:28:10,376 DEBUG SenderThread:877043 [sender.py:send():382] send: telemetry +2024-03-13 11:28:11,058 INFO wandb-upload_0:877043 [upload_job.py:push():131] Uploaded file /tmp/tmpna0izibfwandb/fs37bz2j-wandb-metadata.json +2024-03-13 11:28:11,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_created():271] file/dir created: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:28:13,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:28:15,435 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:16,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:28:20,436 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:25,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:28:25,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:28:25,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:28:26,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:31,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:35,604 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:28:36,652 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:37,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/config.yaml +2024-03-13 11:28:38,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:28:40,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:28:40,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:28:40,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:28:42,372 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:42,792 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:28:42,793 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:28:42,793 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:28:42,797 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:28:43,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_created():271] file/dir created: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:28:44,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:28:46,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:28:46,442 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:28:46,442 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:28:46,448 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:28:47,362 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:28:47,448 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:48,363 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:28:50,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:28:50,239 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:28:50,240 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:28:50,245 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:28:50,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:28:52,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:28:53,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:55,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:28:55,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:28:55,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:28:55,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:28:55,310 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:28:55,310 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:28:55,315 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:28:55,369 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:28:58,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:58,372 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:28:59,447 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:28:59,448 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:28:59,448 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:28:59,453 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:00,374 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:02,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:03,454 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:03,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:03,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:03,789 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:03,794 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:04,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:05,398 DEBUG SystemMonitor:877043 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-13 11:29:05,401 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:29:06,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:09,388 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:09,388 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:09,389 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:09,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:09,393 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:10,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:29:10,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:29:10,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:29:10,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:12,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:13,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:13,796 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:13,796 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:13,801 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:14,391 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:14,837 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:16,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:19,837 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:20,672 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:20,672 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:20,673 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:20,677 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:21,397 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:22,399 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:24,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:24,754 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:24,754 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:24,759 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:25,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:29:25,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:29:25,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:29:25,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:25,401 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:26,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:28,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:28,974 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:28,975 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:28,979 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:29,406 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:30,406 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:31,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:33,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:33,168 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:33,168 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:33,173 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:33,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:35,403 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:29:36,405 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:36,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:36,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:36,940 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:36,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:36,946 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:37,413 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:38,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:40,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:29:40,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:29:40,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:29:42,200 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:42,207 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:42,208 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:42,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:42,213 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:42,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:44,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:47,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:47,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:47,642 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:47,642 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:47,647 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:48,423 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:50,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:52,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:52,021 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:52,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:52,026 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:52,428 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:53,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:54,431 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:55,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:29:55,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:29:55,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:29:56,840 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:56,841 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:29:56,841 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:56,846 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:57,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:29:58,436 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:29:58,885 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:00,687 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:00,688 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:00,688 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:00,694 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:01,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:02,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:04,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:04,280 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:04,280 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:04,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:04,285 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:04,442 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:05,405 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:30:06,445 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:09,407 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:09,739 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:09,740 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:09,740 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:09,745 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:10,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:30:10,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:30:10,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:30:10,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:12,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:14,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:14,058 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:14,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:14,063 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:14,452 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:15,101 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:16,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:20,101 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:22,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:22,320 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:22,321 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:22,326 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:22,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:24,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:25,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:30:25,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:30:25,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:30:25,377 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:26,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:26,023 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:26,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:26,028 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:26,463 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:28,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:30,375 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:30,376 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:30,376 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:30,381 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:30,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:30,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:32,471 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:35,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:35,408 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:30:39,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:39,074 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:39,074 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:39,078 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:39,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:40,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:30:40,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:30:40,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:30:40,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:41,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:42,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:42,482 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:42,483 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:42,487 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:43,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:44,481 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:46,210 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:46,210 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:46,211 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:46,216 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:46,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:47,217 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:48,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:51,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:51,428 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:51,428 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:51,433 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:51,487 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:52,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:54,491 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:30:55,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:30:55,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:30:55,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:30:56,702 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:56,703 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:30:56,703 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:56,708 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:57,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:30:57,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:58,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:01,161 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:01,162 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:01,162 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:01,167 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:01,498 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:03,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:04,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:05,417 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:31:07,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:07,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:07,438 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:07,443 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:07,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:08,443 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:10,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:31:10,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:31:10,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:31:10,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:12,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:12,178 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:12,178 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:12,183 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:12,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:14,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:14,512 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:16,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:16,305 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:16,305 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:16,310 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:16,514 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:18,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:19,311 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:20,742 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:20,743 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:20,743 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:20,749 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:21,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:22,522 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:24,749 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:25,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:31:25,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:31:25,085 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:31:26,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:26,467 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:26,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:26,472 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:26,524 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:28,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:30,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:33,734 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:33,734 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:33,735 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:33,740 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:34,531 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:35,412 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:31:36,413 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:36,532 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:37,479 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:37,480 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:37,480 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:37,486 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:37,532 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:40,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:31:40,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:31:40,085 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:31:40,536 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:41,217 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:41,218 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:41,218 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:41,223 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:41,537 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:42,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:44,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:46,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:46,830 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:46,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:46,836 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:47,542 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:47,836 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:48,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:50,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:50,026 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:50,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:50,032 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:50,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:52,549 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:53,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:53,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:53,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:53,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:53,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:54,551 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:31:55,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:31:55,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:31:55,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:31:56,555 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:31:58,548 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:59,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:59,641 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:31:59,641 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:59,646 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:00,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:02,560 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:32:03,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:04,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:04,567 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:32:04,567 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:04,573 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:05,420 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:32:05,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:06,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:32:09,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:10,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:32:10,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:32:10,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:32:12,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:12,409 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:32:12,409 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:12,414 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:12,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:14,449 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:14,572 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:32:19,449 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:21,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:21,491 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:32:21,492 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:21,497 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:21,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:24,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:24,579 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:32:25,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:32:25,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:32:25,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:32:25,660 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:25,660 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:32:25,660 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:25,665 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:26,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:28,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:32:29,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:31,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:31,364 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:32:31,364 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:31,370 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:31,586 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:34,590 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:32:35,371 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:35,416 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:32:36,301 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:36,302 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:32:36,302 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:36,307 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:36,591 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:38,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:32:40,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:32:40,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:32:40,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:32:40,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:40,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:40,715 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:32:40,715 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:40,720 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:41,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:42,598 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:32:44,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:44,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:32:44,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:44,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:45,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:45,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:46,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:32:49,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:49,578 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:32:49,579 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:49,583 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:49,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:51,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:52,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:32:55,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:32:55,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:32:55,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:32:55,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:55,619 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:32:55,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:55,624 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:56,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:32:56,661 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:58,616 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:00,666 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:00,668 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:00,668 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:00,673 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:01,618 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:01,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:02,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:05,418 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:33:06,108 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:06,109 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:06,109 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:06,114 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:06,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:07,115 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:08,627 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:10,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:33:10,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:33:10,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:33:11,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:11,999 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:11,999 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:12,004 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:12,629 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:13,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:14,632 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:16,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:16,048 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:16,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:16,053 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:16,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:18,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:18,636 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:22,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:22,930 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:22,930 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:22,935 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:23,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:23,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:24,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:25,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:33:25,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:33:25,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:33:28,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:28,730 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:28,731 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:28,736 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:29,645 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:29,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:30,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:32,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:32,590 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:32,590 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:32,595 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:32,649 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:34,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:35,420 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:33:35,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:37,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:37,045 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:37,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:37,050 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:37,655 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:38,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:40,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:33:40,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:33:40,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:33:40,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:40,816 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:40,816 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:40,817 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:40,822 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:41,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:42,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:44,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:44,866 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:44,866 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:44,872 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:45,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:45,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:46,666 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:50,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:52,387 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:52,388 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:52,388 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:52,393 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:52,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:33:54,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:33:55,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:33:55,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:33:55,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:33:56,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:57,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:57,333 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:33:57,333 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:57,338 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:57,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:00,680 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:01,719 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:01,720 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:01,720 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:01,720 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:01,725 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:02,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:04,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:05,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:05,393 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:05,393 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:05,398 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:05,422 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:34:05,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:07,423 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:08,689 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:10,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:34:10,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:34:10,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:34:10,480 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:10,481 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:10,481 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:10,486 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:10,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:12,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:12,693 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:14,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:14,134 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:14,134 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:14,139 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:14,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:16,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:18,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:18,028 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:18,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:18,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:18,034 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:18,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:20,702 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:22,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:22,045 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:22,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:22,050 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:22,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:23,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:24,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:25,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:34:25,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:34:25,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:34:25,608 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:25,608 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:25,609 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:25,614 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:25,708 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:28,649 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:28,711 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:29,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:29,470 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:29,470 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:29,475 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:29,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:32,716 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:33,462 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:33,463 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:33,463 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:33,468 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:33,716 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:34,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:35,429 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:34:36,720 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:38,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:38,195 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:38,195 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:38,202 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:38,723 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:40,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:40,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:34:40,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:34:40,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:34:40,726 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:42,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:42,888 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:42,888 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:42,893 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:43,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:44,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:45,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:46,410 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:46,411 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:46,411 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:46,416 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:46,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:48,733 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:49,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:49,989 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:49,989 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:49,994 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:50,734 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:50,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:52,737 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:53,912 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:53,913 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:53,913 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:53,918 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:54,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:34:55,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:34:55,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:34:55,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:34:56,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:56,743 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:34:59,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:59,500 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:34:59,500 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:59,505 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:59,745 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:01,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:02,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:04,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:04,420 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:04,421 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:04,425 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:04,751 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:05,426 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:35:06,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:07,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:08,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:08,206 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:08,207 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:08,212 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:08,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:10,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:35:10,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:35:10,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:35:10,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:12,492 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:13,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:13,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:13,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:13,997 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:14,759 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:16,762 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:18,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:19,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:19,257 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:19,257 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:19,262 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:19,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:20,767 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:23,196 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:23,197 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:23,197 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:23,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:23,202 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:23,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:24,771 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:25,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:35:25,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:35:25,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:35:28,537 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:32,122 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:32,122 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:32,122 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:32,128 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:32,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:34,165 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:34,779 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:35,433 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:35:35,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:35,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:35,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:35,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:36,780 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:38,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:39,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:40,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:35:40,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:35:40,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:35:41,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:41,793 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:41,794 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:41,799 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:42,787 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:44,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:45,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:45,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:45,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:45,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:45,744 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:45,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:48,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:50,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:51,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:51,578 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:51,578 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:51,583 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:51,796 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:54,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:35:55,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:35:55,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:35:55,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:35:56,513 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:56,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:56,957 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:35:56,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:56,962 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:57,803 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:35:58,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:01,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:01,063 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:01,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:01,069 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:01,807 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:02,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:02,808 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:05,430 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:36:05,454 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:05,454 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:05,455 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:05,459 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:05,810 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:06,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:07,460 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:08,979 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:08,980 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:08,980 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:08,985 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:09,814 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:10,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:36:10,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:36:10,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:36:10,817 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:12,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:12,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:12,949 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:12,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:12,954 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:13,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:14,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:17,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:18,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:18,313 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:18,313 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:18,318 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:18,824 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:20,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:22,920 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:22,921 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:22,921 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:22,926 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:23,829 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:23,927 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:24,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:25,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:36:25,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:36:25,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:36:26,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:26,858 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:26,859 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:26,863 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:27,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:28,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:29,901 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:34,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:35,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:35,115 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:35,115 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:35,120 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:35,438 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:36:35,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:36,843 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:39,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:39,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:39,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:39,516 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:39,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:40,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:36:40,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:36:40,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:36:40,551 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:42,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:44,736 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:44,737 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:44,737 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:44,742 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:44,851 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:45,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:46,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:48,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:48,107 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:48,107 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:48,112 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:48,856 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:50,859 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:36:51,112 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:55,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:36:55,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:36:55,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:36:56,448 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:56,528 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:56,528 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:36:56,528 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:56,533 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:56,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:36:58,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:00,273 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:00,274 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:00,274 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:00,280 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:00,868 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:02,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:02,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:03,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:03,603 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:03,603 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:03,607 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:03,872 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:05,434 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:37:06,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:07,377 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:07,377 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:07,378 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:07,378 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:07,382 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:07,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:08,879 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:10,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:37:10,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:37:10,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:37:11,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:11,261 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:11,261 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:11,267 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:11,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:12,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:13,305 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:16,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:16,209 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:16,210 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:16,215 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:16,885 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:18,886 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:19,216 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:20,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:20,004 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:20,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:20,010 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:20,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:22,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:23,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:23,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:23,695 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:23,700 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:23,891 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:24,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:25,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:37:25,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:37:25,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:37:26,895 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:29,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:29,333 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:29,333 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:29,338 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:29,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:30,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:30,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:33,848 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:33,849 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:33,849 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:33,855 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:33,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:35,437 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:37:35,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:36,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:39,912 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:39,913 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:39,913 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:39,918 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:40,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:37:40,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:37:40,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:37:40,519 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:40,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:42,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:45,520 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:49,107 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:49,108 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:49,108 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:49,113 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:49,916 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:50,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:51,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:52,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:52,970 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:52,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:52,975 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:53,920 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:37:54,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:37:55,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:37:55,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:37:55,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:37:56,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:58,608 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:58,609 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:37:58,609 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:58,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:58,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:00,927 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:01,655 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:03,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:03,054 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:03,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:03,059 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:03,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:04,932 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:05,439 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:38:06,867 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:06,868 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:06,868 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:06,869 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:06,874 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:06,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:08,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:10,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:38:10,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:38:10,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:38:12,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:12,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:12,297 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:12,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:12,302 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:12,940 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:14,943 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:16,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:16,895 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:16,895 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:16,900 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:16,944 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:17,901 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:18,947 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:22,901 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:22,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:22,953 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:22,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:22,958 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:23,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:24,953 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:25,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:38:25,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:38:25,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:38:26,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:26,767 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:26,768 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:26,773 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:26,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:28,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:28,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:30,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:30,622 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:30,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:30,627 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:30,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:32,961 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:34,628 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:35,441 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:38:35,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:35,508 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:35,509 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:35,514 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:35,963 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:36,966 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:39,699 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:39,700 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:39,700 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:39,700 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:39,705 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:39,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:40,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:38:40,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:38:40,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:38:42,973 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:45,523 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:46,168 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:46,169 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:46,169 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:46,174 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:46,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:48,978 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:49,913 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:49,914 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:49,914 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:49,920 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:49,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:50,920 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:52,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:54,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:54,039 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:54,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:54,045 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:54,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:38:55,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:38:55,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:38:55,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:38:56,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:56,988 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:38:57,923 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:57,924 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:38:57,924 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:57,929 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:57,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:00,993 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:01,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:02,926 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:02,927 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:02,927 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:02,933 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:02,994 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:04,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:05,448 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:39:06,525 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:06,526 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:06,526 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:06,531 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:06,999 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:07,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:09,002 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:10,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:39:10,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:39:10,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:39:10,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:10,532 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:10,532 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:10,537 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:11,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:12,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:13,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:16,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:16,383 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:16,384 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:16,389 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:17,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:18,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:19,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:20,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:20,267 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:20,268 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:20,273 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:21,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:23,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:24,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:24,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:24,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:24,808 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:24,814 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:25,018 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:25,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:39:25,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:39:25,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:39:27,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:28,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:28,518 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:28,518 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:28,523 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:29,023 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:29,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:31,026 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:33,198 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:33,199 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:33,199 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:33,204 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:34,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:35,030 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:35,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:35,445 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:39:36,457 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:36,458 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:36,458 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:36,463 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:37,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:39,036 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:40,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:39:40,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:39:40,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:39:40,558 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:40,679 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:40,680 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:40,680 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:40,685 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:41,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:43,040 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:44,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:44,094 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:44,094 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:44,099 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:45,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:46,100 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:47,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:47,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:47,454 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:47,454 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:47,459 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:48,045 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:49,048 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:39:51,460 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:55,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:39:55,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:39:55,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:39:56,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:56,485 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:39:56,485 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:56,485 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:56,490 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:57,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:39:59,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:01,529 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:03,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:03,623 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:03,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:03,628 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:04,060 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:05,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:05,452 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:40:07,454 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:10,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:40:10,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:40:10,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:40:12,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:12,686 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:12,687 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:12,687 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:12,693 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:13,068 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:15,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:17,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:21,122 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:21,122 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:21,123 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:21,128 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:22,075 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:23,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:23,128 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:24,700 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:24,700 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:24,701 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:24,706 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:25,078 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:25,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:40:25,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:40:25,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:40:27,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:28,543 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:29,351 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:29,351 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:29,352 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:29,357 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:30,082 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:31,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:33,333 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:33,334 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:33,334 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:33,339 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:34,087 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:34,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:35,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:35,449 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:40:37,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:37,453 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:37,454 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:37,459 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:38,093 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:39,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:39,459 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:40,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:40:40,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:40:40,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:40:42,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:42,274 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:42,275 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:42,280 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:43,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:45,099 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:45,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:46,778 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:46,779 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:46,779 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:46,785 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:47,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:49,103 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:50,786 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:51,144 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:51,145 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:51,145 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:51,150 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:52,105 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:53,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:40:55,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:40:55,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:40:55,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:40:56,528 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:56,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:56,673 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:40:56,674 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:56,679 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:57,111 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:40:59,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:00,691 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:00,692 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:00,693 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:00,698 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:01,115 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:01,699 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:03,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:05,172 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:05,173 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:05,173 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:05,178 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:05,451 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:41:06,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:07,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:07,452 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:10,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:41:10,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:41:10,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:41:12,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:13,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:13,683 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:13,683 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:13,689 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:14,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:15,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:17,632 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:17,633 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:17,633 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:17,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:17,638 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:18,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:19,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:21,779 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:21,780 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:21,780 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:21,785 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:22,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:22,786 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:23,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:25,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:41:25,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:41:25,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:41:25,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:25,503 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:25,503 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:25,508 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:26,139 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:27,142 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:28,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:30,183 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:30,184 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:30,184 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:30,189 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:31,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:33,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:34,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:34,445 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:34,446 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:34,446 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:34,451 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:35,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:35,453 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:41:37,149 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:38,375 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:38,376 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:38,376 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:38,380 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:39,152 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:39,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:40,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:41:40,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:41:40,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:41:41,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:42,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:42,925 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:42,925 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:42,930 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:43,156 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:44,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:45,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:48,398 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:48,399 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:48,399 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:48,404 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:49,162 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:50,404 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:51,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:52,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:52,718 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:52,718 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:52,723 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:53,166 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:55,169 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:41:55,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:41:55,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:41:55,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:41:55,529 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:56,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:56,705 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:41:56,705 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:56,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:57,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:41:59,175 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:00,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:00,684 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:00,685 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:00,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:00,690 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:01,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:03,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:04,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:04,834 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:04,834 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:04,839 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:05,181 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:05,455 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:42:06,457 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:07,184 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:08,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:08,648 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:08,648 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:08,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:09,185 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:10,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:42:10,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:42:10,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:42:11,189 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:11,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:16,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:16,776 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:16,777 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:16,777 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:16,782 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:17,193 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:19,196 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:21,699 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:21,700 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:21,700 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:21,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:21,705 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:22,198 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:23,200 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:25,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:42:25,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:42:25,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:42:26,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:26,935 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:26,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:26,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:26,940 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:27,203 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:29,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:30,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:30,724 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:30,724 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:30,729 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:31,207 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:32,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:33,210 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:35,462 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:42:36,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:36,118 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:36,118 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:36,123 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:36,212 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:38,123 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:39,217 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:39,854 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:39,854 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:39,855 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:39,860 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:40,217 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:40,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:42:40,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:42:40,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:42:41,218 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:43,544 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:48,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:48,276 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:48,276 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:48,282 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:49,222 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:49,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:51,226 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:52,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:52,828 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:52,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:52,833 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:53,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:54,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:55,230 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:42:55,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:42:55,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:42:55,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:42:57,436 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:57,437 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:42:57,437 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:57,442 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:58,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:42:59,236 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:00,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:02,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:02,724 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:02,724 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:02,729 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:03,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:05,242 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:05,459 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:43:06,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:07,174 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:07,175 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:07,175 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:07,180 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:07,243 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:09,247 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:10,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:43:10,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:43:10,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:43:11,410 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:11,411 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:11,411 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:11,416 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:12,249 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:12,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:13,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:16,318 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:16,319 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:16,319 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:16,324 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:17,254 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:18,324 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:19,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:20,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:20,293 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:20,293 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:20,299 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:21,258 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:23,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:24,299 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:24,479 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:24,480 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:24,480 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:24,486 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:25,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:43:25,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:43:25,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:43:25,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:27,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:29,551 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:29,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:29,691 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:29,691 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:29,696 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:30,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:31,270 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:34,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:35,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:35,340 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:35,341 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:35,346 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:35,467 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:43:36,273 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:37,275 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:39,344 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:39,345 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:39,345 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:39,350 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:40,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:43:40,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:43:40,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:43:40,278 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:40,506 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:41,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:43,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:43,038 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:43,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:43,044 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:43,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:45,285 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:46,081 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:48,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:48,831 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:48,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:48,837 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:49,287 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:51,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:51,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:53,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:53,404 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:53,404 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:53,410 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:54,290 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:43:55,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:43:55,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:43:55,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:43:55,293 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:43:57,526 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:58,105 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:58,106 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:43:58,106 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:58,111 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:58,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:01,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:03,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:03,026 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:03,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:03,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:03,031 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:03,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:05,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:05,464 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:44:07,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:07,094 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:07,094 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:07,100 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:07,307 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:08,100 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:09,310 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:10,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:44:10,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:44:10,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:44:11,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:11,354 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:11,354 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:11,359 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:12,312 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:13,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:13,397 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:15,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:15,911 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:15,911 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:15,916 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:16,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:17,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:18,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:20,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:20,459 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:20,459 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:20,465 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:21,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:23,325 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:24,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:25,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:25,040 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:25,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:25,045 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:25,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:44:25,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:44:25,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:44:25,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:27,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:28,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:28,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:28,808 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:28,814 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:29,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:29,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:31,332 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:33,401 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:33,401 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:33,402 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:33,407 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:34,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:35,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:35,407 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:35,471 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:44:37,401 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:37,402 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:37,402 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:37,407 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:38,339 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:39,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:40,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:44:40,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:44:40,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:44:40,480 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:41,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:41,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:41,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:41,937 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:42,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:43,345 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:45,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:46,196 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:46,197 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:46,197 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:46,202 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:46,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:49,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:50,359 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:50,360 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:50,360 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:50,365 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:51,351 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:51,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:53,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:44:54,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:54,271 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:44:54,271 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:54,276 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:54,355 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:44:55,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:44:55,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:44:55,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:44:56,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:57,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:01,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:03,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:03,484 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:03,485 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:03,490 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:04,363 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:05,366 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:05,468 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:45:07,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:08,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:08,281 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:08,281 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:08,286 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:08,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:10,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:45:10,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:45:10,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:45:11,372 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:12,536 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:15,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:15,032 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:15,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:15,038 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:15,374 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:17,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:18,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:18,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:18,747 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:18,747 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:18,752 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:19,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:21,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:22,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:22,922 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:22,922 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:22,928 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:23,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:23,928 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:25,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:45:25,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:45:25,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:45:25,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:29,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:31,161 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:31,161 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:31,162 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:31,167 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:31,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:33,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:35,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:35,470 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:45:35,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:35,962 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:35,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:35,968 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:36,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:37,397 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:40,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:45:40,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:45:40,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:45:40,542 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:40,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:40,936 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:40,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:40,943 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:41,400 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:43,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:45,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:45,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:45,296 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:45,301 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:45,405 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:46,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:47,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:51,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:51,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:51,445 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:51,445 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:51,450 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:52,411 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:53,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:45:55,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:45:55,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:45:55,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:45:55,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:55,536 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:45:55,536 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:55,541 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:56,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:45:56,542 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:57,420 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:01,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:01,014 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:01,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:01,020 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:01,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:02,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:03,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:04,719 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:04,720 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:04,720 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:04,725 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:05,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:05,472 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:46:07,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:07,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:09,228 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:09,229 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:09,229 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:09,234 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:09,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:10,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:46:10,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:46:10,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:46:11,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:12,524 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:13,079 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:13,080 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:13,080 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:13,085 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:13,437 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:15,440 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:16,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:16,868 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:16,869 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:16,873 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:17,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:17,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:19,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:21,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:21,582 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:21,582 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:21,587 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:22,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:23,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:23,588 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:25,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:46:25,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:46:25,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:46:26,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:26,012 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:26,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:26,018 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:26,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:27,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:29,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:30,134 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:30,135 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:30,135 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:30,140 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:30,453 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:31,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:34,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:35,474 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:46:38,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:38,649 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:38,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:38,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:39,461 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:39,655 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:40,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:46:40,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:46:40,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:46:41,463 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:43,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:43,340 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:43,340 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:43,345 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:43,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:45,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:45,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:46,863 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:46,864 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:46,864 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:46,869 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:47,468 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:49,471 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:50,630 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:50,631 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:50,631 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:50,632 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:50,637 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:51,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:53,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:54,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:54,306 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:54,307 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:54,312 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:54,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:46:55,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:46:55,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:46:55,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:46:56,475 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:57,481 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:46:58,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:58,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:46:58,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:58,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:59,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:01,486 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:01,781 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:02,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:02,453 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:02,454 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:02,459 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:02,486 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:05,481 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:47:05,491 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:06,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:06,549 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:06,550 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:06,555 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:07,492 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:07,555 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:09,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:10,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:47:10,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:47:10,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:47:10,684 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:10,685 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:10,685 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:10,691 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:11,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:12,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:13,501 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:17,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:19,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:19,036 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:19,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:19,041 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:19,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:21,507 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:23,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:25,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:25,243 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:25,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:25,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:47:25,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:47:25,248 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:25,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:47:25,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:27,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:28,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:33,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:34,282 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:34,283 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:34,283 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:34,288 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:34,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:35,478 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:47:37,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:38,432 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:38,433 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:38,433 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:38,438 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:38,522 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:39,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:40,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:47:40,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:47:40,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:47:41,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:44,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:46,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:46,575 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:46,575 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:46,580 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:47,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:49,533 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:49,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:50,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:50,746 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:50,747 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:50,752 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:51,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:53,537 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:54,349 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:54,350 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:54,350 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:54,355 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:54,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:47:55,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:47:55,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:47:55,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:47:55,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:57,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:47:58,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:58,634 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:47:58,634 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:58,639 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:59,544 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:00,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:01,547 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:02,343 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:02,343 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:02,344 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:02,349 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:02,548 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:05,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:05,590 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:48:06,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:07,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:07,293 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:07,294 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:07,299 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:07,590 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:09,593 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:10,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:48:10,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:48:10,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:48:12,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:12,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:12,516 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:12,516 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:12,522 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:12,595 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:15,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:16,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:16,239 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:16,239 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:16,244 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:16,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:17,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:18,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:19,749 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:19,750 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:19,750 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:19,755 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:20,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:21,604 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:23,756 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:25,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:48:25,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:48:25,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:48:25,502 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:25,536 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:25,536 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:25,542 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:25,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:27,610 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:29,542 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:31,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:31,028 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:31,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:31,033 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:31,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:33,616 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:35,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:35,482 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:48:36,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:36,008 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:36,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:36,013 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:36,618 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:37,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:39,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:39,575 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:39,576 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:39,581 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:39,622 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:40,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:48:40,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:48:40,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:48:40,446 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:41,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:44,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:44,831 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:44,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:44,836 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:45,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:45,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:47,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:50,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:50,510 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:50,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:50,516 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:50,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:51,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:53,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:55,217 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:55,218 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:55,219 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:55,224 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:55,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:48:55,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:48:55,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:48:55,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:48:57,464 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:57,643 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:48:58,760 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:58,761 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:48:58,761 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:58,766 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:59,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:01,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:02,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:02,747 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:02,747 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:02,747 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:02,752 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:03,649 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:05,484 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:49:05,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:08,486 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:10,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:49:10,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:49:10,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:49:11,080 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:11,081 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:11,081 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:11,087 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:11,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:13,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:14,125 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:15,232 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:15,234 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:15,234 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:15,240 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:15,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:17,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:19,081 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:19,082 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:19,082 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:19,087 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:19,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:20,088 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:21,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:24,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:24,079 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:24,079 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:24,084 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:24,668 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:25,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:49:25,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:49:25,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:49:25,479 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:25,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:27,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:27,999 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:27,999 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:28,005 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:28,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:29,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:31,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:34,401 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:34,402 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:34,402 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:34,407 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:34,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:35,486 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:49:36,488 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:37,683 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:38,349 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:38,350 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:38,351 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:38,356 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:38,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:39,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:40,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:49:40,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:49:40,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:49:41,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:43,610 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:43,611 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:43,611 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:43,616 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:43,691 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:45,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:46,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:47,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:47,961 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:47,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:47,967 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:48,696 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:49,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:51,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:53,495 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:53,496 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:53,497 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:53,502 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:53,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:55,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:49:55,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:49:55,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:49:55,705 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:49:57,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:57,728 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:57,729 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:49:57,729 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:57,734 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:58,708 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:49:59,711 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:02,426 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:02,427 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:02,427 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:02,433 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:02,713 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:03,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:05,488 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:50:05,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:06,368 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:06,368 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:06,369 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:06,374 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:06,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:07,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:09,375 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:10,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:50:10,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:50:10,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:50:10,288 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:10,526 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:10,526 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:10,531 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:10,720 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:13,723 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:14,498 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:14,499 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:14,500 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:14,500 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:14,505 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:14,723 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:17,727 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:18,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:18,986 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:18,986 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:18,991 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:19,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:19,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:21,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:24,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:24,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:24,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:24,939 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:25,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:50:25,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:50:25,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:50:25,472 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:25,734 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:27,737 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:29,311 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:29,312 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:29,312 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:29,318 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:29,739 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:31,354 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:31,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:33,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:33,608 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:33,608 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:33,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:33,743 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:35,494 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:50:35,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:36,495 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:38,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:38,156 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:38,157 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:38,161 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:38,750 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:39,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:40,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:50:40,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:50:40,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:50:42,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:42,374 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:42,374 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:42,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:42,379 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:42,754 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:43,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:47,418 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:50,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:50,539 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:50,539 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:50,544 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:50,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:52,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:53,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:54,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:54,564 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:54,564 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:54,570 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:54,765 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:50:55,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:50:55,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:50:55,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:50:57,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:50:58,440 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:58,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:58,458 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:50:58,458 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:58,463 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:58,771 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:01,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:03,502 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:03,661 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:03,662 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:03,662 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:03,667 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:03,776 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:05,493 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:51:05,779 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:07,986 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:07,987 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:07,987 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:07,992 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:08,781 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:08,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:09,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:10,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:51:10,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:51:10,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:51:11,634 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:11,634 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:11,634 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:11,639 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:11,785 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:13,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:14,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:15,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:15,577 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:15,577 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:15,582 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:15,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:17,793 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:19,756 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:19,757 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:19,757 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:19,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:19,762 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:19,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:21,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:23,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:23,434 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:23,434 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:23,439 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:23,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:25,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:51:25,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:51:25,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:51:25,533 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:25,800 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:29,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:29,648 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:29,648 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:29,653 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:29,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:30,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:31,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:35,498 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:51:35,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:35,682 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:35,683 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:35,687 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:35,808 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:36,688 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:37,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:40,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:40,240 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:40,240 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:40,245 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:40,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:51:40,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:51:40,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:51:40,814 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:41,815 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:42,511 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:46,074 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:46,075 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:46,075 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:46,080 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:46,818 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:47,818 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:48,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:50,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:50,867 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:50,867 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:50,872 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:51,821 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:53,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:53,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:55,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:51:55,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:51:55,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:51:56,269 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:56,270 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:51:56,270 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:56,275 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:56,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:51:57,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:51:59,313 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:00,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:00,043 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:00,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:00,049 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:00,830 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:52:01,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:52:04,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:04,066 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:04,067 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:04,071 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:04,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:52:05,072 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:05,496 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:52:05,837 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:52:07,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:07,873 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:07,873 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:07,878 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:08,839 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:52:09,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:52:10,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:52:10,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:52:10,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:52:10,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:12,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:12,649 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:12,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:12,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:12,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:52:15,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:15,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:52:17,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:17,485 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:17,485 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:17,490 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:17,847 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:52:19,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:52:21,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:22,189 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:22,190 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:22,190 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:22,196 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:22,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:52:23,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:52:25,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:52:25,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:52:25,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:52:26,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:26,431 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:26,431 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:26,436 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:26,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:52:27,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:27,860 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:52:32,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:33,350 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:33,351 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:33,351 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:33,356 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:33,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:52:35,498 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:52:35,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:52:37,500 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:40,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:52:40,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:52:40,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:52:42,432 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:42,433 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:42,433 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:42,438 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:42,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:52:43,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:43,875 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:52:48,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:50,815 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:50,816 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:50,816 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:50,821 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:50,879 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:52:53,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:53,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:52:55,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:52:55,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:52:55,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:52:59,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:59,222 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:52:59,222 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:59,222 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:59,227 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:59,886 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:01,889 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:04,269 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:04,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:04,854 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:04,854 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:04,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:04,891 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:05,501 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:53:07,895 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:09,502 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:10,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:53:10,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:53:10,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:53:13,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:13,209 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:13,209 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:13,214 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:13,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:15,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:15,900 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:17,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:17,230 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:17,231 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:17,236 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:17,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:19,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:20,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:20,797 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:20,798 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:20,798 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:20,802 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:20,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:23,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:25,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:53:25,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:53:25,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:53:25,586 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:25,586 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:25,587 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:25,592 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:25,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:26,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:27,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:30,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:30,375 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:30,375 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:30,380 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:30,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:31,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:32,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:34,927 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:34,927 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:34,928 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:34,932 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:35,503 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:53:35,920 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:37,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:37,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:40,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:53:40,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:53:40,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:53:40,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:40,487 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:40,488 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:40,493 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:40,926 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:41,928 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:43,494 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:45,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:45,320 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:45,320 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:45,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:45,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:47,934 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:49,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:49,379 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:49,380 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:49,380 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:49,385 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:49,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:51,938 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:54,386 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:54,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:54,995 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:54,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:55,000 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:55,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:53:55,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:53:55,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:53:55,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:53:57,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:53:59,475 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:59,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:59,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:53:59,853 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:59,858 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:59,947 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:01,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:04,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:04,061 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:04,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:04,067 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:04,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:05,067 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:05,508 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:54:05,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:09,423 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:09,424 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:09,424 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:09,429 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:09,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:10,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:54:10,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:54:10,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:54:10,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:11,961 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:15,539 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:18,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:18,554 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:18,554 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:18,560 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:18,965 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:19,968 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:20,598 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:23,144 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:23,145 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:23,145 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:23,151 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:23,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:25,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:54:25,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:54:25,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:54:25,974 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:26,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:26,712 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:26,713 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:26,713 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:26,718 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:26,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:29,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:31,754 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:32,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:32,996 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:32,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:33,002 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:33,981 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:35,507 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:54:35,985 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:37,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:37,428 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:37,428 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:37,429 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:37,434 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:37,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:39,991 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:40,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:54:40,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:54:40,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:54:42,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:45,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:45,915 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:45,915 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:45,920 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:45,995 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:47,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:47,999 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:50,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:50,255 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:50,256 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:50,261 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:51,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:52,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:53,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:54,076 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:54,076 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:54,077 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:54,082 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:55,005 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:54:55,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:54:55,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:54:55,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:54:56,008 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:54:58,537 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:58,639 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:58,640 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:54:58,640 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:58,645 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:59,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:00,014 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:02,665 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:02,666 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:02,666 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:02,671 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:03,016 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:03,671 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:04,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:05,512 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:55:06,558 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:06,559 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:06,559 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:06,564 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:07,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:08,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:09,565 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:10,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:55:10,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:55:10,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:55:10,365 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:10,585 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:10,585 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:10,590 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:11,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:12,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:14,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:15,462 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:15,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:15,463 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:15,468 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:16,029 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:18,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:20,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:21,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:21,911 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:21,911 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:21,916 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:22,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:24,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:25,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:55:25,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:55:25,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:55:25,525 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:26,867 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:26,868 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:26,868 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:26,874 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:27,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:30,045 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:30,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:31,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:31,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:31,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:31,473 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:32,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:34,050 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:35,511 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:55:36,513 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:37,375 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:37,376 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:37,376 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:37,381 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:38,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:40,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:40,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:55:40,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:55:40,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:55:41,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:41,982 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:41,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:41,983 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:41,988 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:42,058 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:44,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:45,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:45,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:45,990 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:45,995 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:46,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:46,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:48,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:51,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:51,254 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:51,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:51,260 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:52,068 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:52,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:54,071 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:55,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:55:55,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:55:55,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:55:55,418 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:55,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:55,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:55,516 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:56,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:55:57,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:58,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:55:58,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:58,996 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:55:58,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:59,001 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:59,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:02,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:03,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:03,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:03,935 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:03,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:03,943 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:04,082 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:05,514 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:56:06,088 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:08,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:09,178 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:09,179 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:09,179 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:09,185 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:10,092 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:10,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:56:10,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:56:10,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:56:12,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:13,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:13,954 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:13,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:13,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:13,961 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:14,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:16,103 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:18,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:18,454 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:18,454 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:18,470 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:19,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:19,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:20,111 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:22,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:22,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:22,278 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:22,284 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:23,113 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:24,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:25,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:56:25,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:56:25,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:56:25,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:25,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:25,954 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:25,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:25,961 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:26,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:28,129 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:29,705 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:29,706 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:29,707 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:29,713 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:30,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:30,713 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:32,136 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:33,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:33,716 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:33,716 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:33,722 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:34,138 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:35,516 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:56:36,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:36,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:39,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:39,064 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:39,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:39,074 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:39,143 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:40,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:56:40,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:56:40,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:56:42,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:42,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:43,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:43,474 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:43,474 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:43,482 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:44,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:46,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:47,518 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:49,713 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:49,714 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:49,714 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:49,723 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:50,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:52,162 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:52,724 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:55,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:56:55,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:56:55,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:56:55,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:55,554 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:56:55,555 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:55,562 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:56,167 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:56:57,173 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:56:58,598 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:01,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:01,827 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:01,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:01,833 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:02,177 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:03,180 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:03,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:05,518 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:57:07,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:07,679 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:07,680 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:07,687 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:08,185 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:09,190 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:09,688 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:10,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:57:10,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:57:10,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:57:12,879 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:12,880 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:12,880 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:12,885 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:13,194 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:14,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:15,199 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:17,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:17,354 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:17,354 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:17,360 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:18,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:19,205 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:20,360 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:21,906 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:21,907 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:21,908 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:21,914 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:22,208 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:23,213 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:25,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:57:25,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:57:25,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:57:25,506 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:26,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:26,831 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:26,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:26,837 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:27,221 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:27,221 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:30,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:31,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:31,510 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:31,510 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:31,517 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:32,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:33,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:35,525 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:57:36,526 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:37,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:37,727 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:37,727 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:37,735 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:38,231 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:39,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:40,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:57:40,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:57:40,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:57:41,543 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:45,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:45,911 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:45,911 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:45,918 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:46,244 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:46,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:47,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:50,067 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:50,067 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:50,068 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:50,075 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:50,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:51,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:52,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:54,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:54,263 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:54,263 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:54,272 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:55,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:57:55,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:57:55,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:57:55,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:55,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:57:57,543 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:58,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:58,028 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:57:58,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:58,034 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:58,266 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:57:59,272 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:03,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:03,269 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:03,270 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:03,270 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:03,275 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:03,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:05,281 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:05,522 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:58:07,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:07,024 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:07,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:07,032 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:07,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:09,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:09,287 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:10,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:58:10,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:58:10,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:58:11,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:11,009 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:11,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:11,016 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:11,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:13,295 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:14,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:15,214 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:15,215 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:15,216 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:15,222 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:15,297 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:17,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:19,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:20,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:20,054 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:20,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:20,060 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:20,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:21,307 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:24,364 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:24,365 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:24,365 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:24,365 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:24,371 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:25,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:58:25,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:58:25,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:58:25,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:25,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:28,598 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:28,599 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:28,599 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:28,605 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:29,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:29,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:29,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:32,375 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:32,376 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:32,376 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:32,382 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:33,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:33,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:35,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:35,530 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:58:36,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:36,539 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:36,539 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:36,546 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:37,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:37,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:40,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:58:40,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:58:40,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:58:40,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:40,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:40,621 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:40,621 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:40,627 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:41,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:41,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:45,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:45,539 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:45,539 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:45,545 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:46,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:46,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:47,344 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:51,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:53,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:53,960 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:58:53,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:53,965 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:54,349 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:58:55,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:58:55,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:58:55,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:58:55,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:58:56,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:01,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:02,343 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:02,344 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:59:02,344 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:02,350 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:02,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:59:03,365 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:59:05,526 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:59:06,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:06,008 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:59:06,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:06,014 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:06,368 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:59:07,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:07,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:59:10,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:59:10,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:59:10,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:59:11,586 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:11,587 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:59:11,587 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:11,594 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:12,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:59:12,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:13,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:59:16,095 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:16,096 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:59:16,096 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:16,102 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:16,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:59:17,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:59:18,103 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:19,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:19,758 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:59:19,759 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:19,765 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:20,388 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:59:21,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:59:23,766 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:25,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:59:25,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:59:25,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:59:25,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:25,514 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:59:25,515 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:25,520 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:26,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:59:27,400 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:59:29,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:29,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:29,824 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:59:29,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:29,830 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:30,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:59:31,405 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:59:34,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:34,420 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:59:34,420 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:34,426 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:35,411 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:59:35,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:59:35,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:35,528 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 11:59:40,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:59:40,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:59:40,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:59:40,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:44,388 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:44,389 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:59:44,389 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:44,395 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:44,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:59:45,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:59:46,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:51,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:52,396 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:52,397 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 11:59:52,397 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:52,402 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:52,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 11:59:53,433 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 11:59:55,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:59:55,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:59:55,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:59:56,507 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:00,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:00,617 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:00,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:00,622 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:01,442 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:01,442 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:01,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:04,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:04,881 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:04,881 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:04,886 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:05,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:05,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:05,531 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:00:07,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:10,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:00:10,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:00:10,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:00:10,472 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:10,590 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:10,590 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:10,596 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:11,453 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:11,453 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:12,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:14,630 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:14,631 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:14,631 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:14,636 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:15,458 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:15,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:17,637 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:18,196 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:18,197 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:18,197 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:18,203 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:18,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:19,463 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:21,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:21,995 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:21,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:22,001 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:22,465 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:23,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:23,468 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:25,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:00:25,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:00:25,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:00:25,585 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:25,586 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:25,586 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:25,591 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:26,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:27,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:28,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:29,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:29,290 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:29,291 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:29,296 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:29,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:31,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:34,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:34,443 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:34,444 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:34,444 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:34,449 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:34,479 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:35,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:35,533 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:00:39,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:40,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:00:40,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:00:40,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:00:40,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:40,595 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:40,596 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:40,601 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:41,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:41,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:44,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:46,918 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:46,919 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:46,919 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:46,924 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:47,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:47,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:49,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:50,724 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:50,725 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:50,725 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:50,730 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:51,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:51,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:00:54,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:54,757 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:00:54,758 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:54,764 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:55,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:00:55,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:00:55,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:00:55,503 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:00:55,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:55,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:00,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:00,870 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:00,870 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:00,871 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:00,876 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:01,511 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:01,511 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:04,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:04,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:04,512 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:04,517 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:05,515 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:05,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:05,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:05,538 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:01:08,912 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:08,913 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:08,913 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:08,918 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:09,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:09,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:10,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:01:10,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:01:10,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:01:10,884 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:12,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:12,954 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:12,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:12,960 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:13,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:13,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:15,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:17,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:17,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:17,697 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:17,702 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:18,529 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:19,531 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:21,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:23,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:23,569 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:23,570 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:23,575 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:24,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:25,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:01:25,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:01:25,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:01:25,537 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:27,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:27,794 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:27,795 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:27,795 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:27,800 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:28,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:29,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:31,776 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:31,777 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:31,777 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:31,782 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:32,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:32,782 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:33,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:35,537 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:01:37,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:37,332 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:37,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:37,338 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:37,548 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:38,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:39,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:40,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:01:40,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:01:40,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:01:43,519 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:45,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:45,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:45,808 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:45,814 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:46,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:47,560 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:48,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:53,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:54,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:54,118 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:54,119 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:54,123 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:54,564 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:55,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:01:55,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:01:55,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:01:55,567 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:01:58,426 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:58,427 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:01:58,427 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:58,432 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:58,570 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:01:59,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:59,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:02,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:02,196 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:02,196 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:02,202 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:02,575 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:03,578 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:05,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:05,542 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:02:05,860 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:05,861 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:05,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:05,866 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:06,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:07,583 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:10,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:02:10,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:02:10,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:02:10,425 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:10,922 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:10,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:10,922 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:10,927 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:11,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:11,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:15,928 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:19,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:19,997 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:19,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:20,002 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:20,593 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:21,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:21,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:23,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:23,324 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:23,324 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:23,329 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:23,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:25,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:02:25,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:02:25,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:02:25,601 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:26,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:27,462 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:27,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:27,463 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:27,468 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:27,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:29,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:31,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:31,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:31,512 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:31,517 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:31,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:32,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:33,610 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:35,542 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:02:35,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:35,547 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:35,547 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:35,552 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:35,611 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:37,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:37,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:39,699 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:39,700 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:39,700 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:39,705 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:40,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:02:40,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:02:40,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:02:40,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:41,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:43,459 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:43,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:43,466 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:43,466 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:43,472 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:43,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:45,624 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:48,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:49,494 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:49,495 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:49,495 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:49,500 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:49,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:51,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:02:54,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:55,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:02:55,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:02:55,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:02:58,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:58,016 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:02:58,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:58,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:58,636 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:02:59,638 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:00,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:01,736 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:01,737 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:01,737 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:01,742 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:02,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:03,643 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:05,544 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:03:05,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:07,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:07,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:07,258 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:07,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:07,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:07,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:10,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:03:10,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:03:10,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:03:11,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:11,295 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:11,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:11,295 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:11,301 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:11,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:11,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:15,837 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:15,838 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:15,838 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:15,843 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:16,655 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:16,843 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:17,658 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:21,844 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:24,608 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:24,609 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:24,609 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:24,614 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:24,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:25,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:03:25,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:03:25,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:03:25,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:27,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:28,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:28,944 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:28,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:28,950 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:29,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:29,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:32,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:33,897 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:33,898 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:33,898 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:33,903 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:34,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:35,546 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:03:35,674 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:37,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:37,601 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:37,601 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:37,606 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:37,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:38,606 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:39,680 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:40,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:03:40,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:03:40,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:03:42,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:42,004 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:42,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:42,009 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:42,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:43,683 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:44,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:46,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:46,922 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:46,923 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:46,928 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:47,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:47,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:49,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:51,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:51,814 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:51,814 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:51,819 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:52,691 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:53,693 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:03:55,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:03:55,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:03:55,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:03:55,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:56,779 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:56,780 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:03:56,780 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:56,785 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:57,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:03:57,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:00,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:00,649 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:00,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:00,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:00,655 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:00,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:01,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:05,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:05,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:05,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:05,473 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:05,548 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:04:05,706 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:06,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:07,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:09,100 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:09,100 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:09,101 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:09,106 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:09,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:09,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:10,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:04:10,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:04:10,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:04:12,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:13,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:13,238 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:13,238 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:13,243 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:13,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:13,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:17,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:17,422 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:17,423 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:17,424 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:17,429 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:17,719 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:19,722 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:21,368 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:21,369 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:21,370 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:21,375 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:21,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:21,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:22,375 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:25,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:04:25,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:04:25,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:04:26,192 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:26,193 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:26,193 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:26,198 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:26,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:27,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:28,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:31,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:31,198 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:31,198 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:31,203 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:31,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:31,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:34,204 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:35,550 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:04:35,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:35,618 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:35,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:35,623 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:35,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:37,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:39,624 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:40,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:04:40,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:04:40,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:04:44,749 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:44,750 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:44,750 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:44,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:44,755 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:45,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:45,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:49,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:50,796 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:50,796 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:50,797 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:50,802 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:51,754 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:51,754 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:54,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:54,606 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:54,606 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:54,612 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:54,755 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:55,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:04:55,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:04:55,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:04:55,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:55,758 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:04:58,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:58,561 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:04:58,562 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:58,567 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:58,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:04:59,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:00,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:03,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:03,297 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:03,297 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:03,302 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:03,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:03,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:05,552 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:05:06,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:07,672 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:07,673 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:07,673 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:07,678 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:07,771 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:09,774 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:10,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:05:10,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:05:10,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:05:12,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:12,740 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:12,741 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:12,741 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:12,746 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:12,776 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:13,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:17,159 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:17,160 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:17,161 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:17,166 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:17,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:17,783 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:18,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:21,425 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:21,426 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:21,426 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:21,431 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:21,787 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:21,787 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:23,431 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:25,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:05:25,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:05:25,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:05:26,435 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:26,436 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:26,436 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:26,441 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:26,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:27,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:28,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:30,760 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:30,760 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:30,761 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:30,766 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:30,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:31,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:33,766 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:34,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:34,279 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:34,279 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:34,286 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:34,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:35,560 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:05:35,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:39,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:39,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:39,996 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:39,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:40,002 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:40,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:05:40,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:05:40,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:05:40,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:41,807 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:43,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:43,626 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:43,626 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:43,632 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:43,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:44,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:45,812 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:49,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:51,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:51,583 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:51,583 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:51,588 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:51,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:53,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:05:55,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:05:55,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:05:55,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:05:55,537 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:55,634 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:55,635 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:05:55,635 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:55,640 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:55,820 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:05:57,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:00,147 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:00,148 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:00,149 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:00,154 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:00,827 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:01,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:01,829 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:05,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:05,076 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:05,076 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:05,081 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:05,556 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:06:05,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:05,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:06,557 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:10,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:06:10,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:06:10,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:06:10,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:10,973 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:10,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:10,979 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:11,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:11,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:12,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:15,391 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:15,392 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:15,392 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:15,398 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:15,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:15,845 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:17,398 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:19,378 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:19,379 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:19,379 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:19,384 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:19,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:19,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:23,384 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:23,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:23,935 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:23,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:23,940 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:24,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:25,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:06:25,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:06:25,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:06:25,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:28,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:32,399 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:32,400 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:32,400 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:32,405 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:32,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:33,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:34,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:35,558 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:06:36,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:36,997 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:36,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:37,002 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:37,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:37,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:40,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:40,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:06:40,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:06:40,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:06:42,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:42,470 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:42,470 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:42,475 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:42,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:43,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:45,513 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:47,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:47,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:47,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:47,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:47,875 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:49,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:50,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:53,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:53,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:06:53,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:53,988 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:54,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:06:55,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:06:55,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:06:55,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:06:55,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:06:56,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:01,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:02,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:02,409 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:02,409 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:02,414 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:02,889 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:03,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:05,560 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:07:06,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:06,226 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:06,226 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:06,231 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:06,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:07,232 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:07,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:10,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:07:10,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:07:10,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:07:11,836 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:11,836 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:11,837 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:11,842 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:11,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:12,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:13,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:15,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:15,835 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:15,836 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:15,840 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:15,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:17,906 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:18,841 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:19,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:19,575 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:19,575 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:19,580 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:19,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:19,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:24,415 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:24,416 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:24,416 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:24,416 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:24,421 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:24,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:25,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:07:25,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:07:25,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:07:25,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:28,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:28,691 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:28,691 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:28,697 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:28,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:29,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:29,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:33,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:33,017 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:33,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:33,022 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:33,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:33,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:35,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:35,562 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:07:39,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:39,500 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:39,500 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:39,504 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:39,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:39,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:40,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:07:40,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:07:40,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:07:40,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:42,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:42,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:42,789 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:42,794 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:42,932 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:43,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:45,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:46,675 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:46,676 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:46,676 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:46,681 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:46,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:47,939 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:50,867 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:50,868 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:50,868 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:50,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:50,873 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:50,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:51,944 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:07:55,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:07:55,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:07:55,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:07:55,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:55,915 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:07:55,915 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:55,915 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:55,920 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:55,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:07:57,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:00,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:01,551 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:01,552 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:01,552 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:01,558 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:01,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:01,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:05,564 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:08:06,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:07,876 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:07,877 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:07,877 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:07,882 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:07,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:09,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:10,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:08:10,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:08:10,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:08:12,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:13,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:13,057 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:13,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:13,062 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:13,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:13,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:18,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:19,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:19,768 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:19,768 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:19,774 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:19,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:21,973 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:23,774 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:25,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:25,117 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:25,118 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:25,122 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:25,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:08:25,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:08:25,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:08:25,978 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:25,978 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:29,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:29,172 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:29,172 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:29,172 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:29,177 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:29,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:29,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:32,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:32,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:32,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:32,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:32,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:33,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:34,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:35,566 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:08:37,324 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:37,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:37,325 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:37,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:37,990 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:37,991 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:40,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:08:40,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:08:40,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:08:40,478 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:43,359 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:43,360 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:43,360 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:43,365 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:43,997 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:43,997 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:46,401 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:50,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:50,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:50,739 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:50,744 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:51,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:51,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:52,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:55,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:08:55,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:08:55,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:08:55,416 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:55,514 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:55,514 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:55,519 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:56,008 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:08:56,008 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:08:57,520 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:59,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:59,993 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:08:59,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:59,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:00,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:02,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:02,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:05,572 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:09:06,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:06,224 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:06,224 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:06,229 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:07,018 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:08,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:08,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:10,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:09:10,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:09:10,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:09:10,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:10,560 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:10,560 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:10,565 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:11,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:12,025 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:13,601 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:16,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:16,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:16,522 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:16,527 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:17,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:18,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:19,527 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:20,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:20,577 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:20,577 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:20,583 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:21,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:22,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:24,583 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:24,588 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:24,589 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:24,589 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:24,594 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:25,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:25,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:09:25,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:09:25,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:09:26,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:28,857 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:28,858 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:28,858 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:28,863 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:29,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:29,901 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:30,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:34,273 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:34,274 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:34,274 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:34,279 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:35,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:35,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:35,571 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:09:36,048 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:39,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:39,603 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:39,604 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:39,608 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:40,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:40,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:40,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:09:40,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:09:40,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:09:40,479 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:45,479 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:45,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:45,757 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:45,758 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:45,763 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:46,058 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:48,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:50,801 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:51,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:51,546 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:51,546 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:51,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:52,065 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:52,065 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:09:55,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:09:55,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:09:55,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:09:56,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:56,637 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:56,637 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:09:56,638 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:56,642 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:57,068 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:09:58,071 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:00,404 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:00,405 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:00,405 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:00,410 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:01,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:02,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:02,410 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:05,576 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:10:05,635 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:05,636 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:05,636 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:05,641 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:06,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:06,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:07,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:09,109 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:09,110 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:09,110 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:09,116 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:10,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:10,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:10,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:10:10,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:10:10,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:10:13,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:13,700 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:13,700 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:13,700 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:13,705 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:14,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:14,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:18,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:18,815 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:18,815 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:18,815 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:18,820 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:19,093 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:20,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:23,593 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:23,593 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:23,594 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:23,598 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:24,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:24,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:24,599 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:25,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:10:25,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:10:25,930 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:10:28,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:28,686 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:28,686 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:28,691 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:29,103 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:29,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:30,105 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:32,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:32,621 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:32,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:32,626 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:33,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:34,109 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:35,575 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:10:35,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:37,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:37,422 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:37,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:37,428 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:38,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:38,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:40,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:10:40,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:10:40,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:10:41,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:44,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:44,048 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:44,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:44,053 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:44,118 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:46,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:47,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:52,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:52,544 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:52,545 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:52,545 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:52,550 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:53,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:54,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:10:55,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:10:55,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:10:55,930 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:10:57,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:57,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:57,210 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:10:57,210 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:57,215 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:58,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:10:58,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:00,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:00,875 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:00,875 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:00,880 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:01,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:02,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:02,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:05,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:05,523 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:05,523 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:05,528 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:05,577 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:11:06,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:06,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:08,578 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:10,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:11:10,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:11:10,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:11:11,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:11,170 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:11,171 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:11,175 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:12,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:12,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:14,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:16,183 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:16,184 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:16,184 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:16,189 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:17,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:18,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:19,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:20,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:20,960 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:20,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:20,965 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:21,153 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:22,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:24,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:25,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:25,218 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:25,219 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:25,223 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:25,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:11:25,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:11:25,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:11:26,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:26,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:29,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:29,849 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:29,849 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:29,854 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:30,161 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:30,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:32,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:33,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:33,367 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:33,367 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:33,372 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:34,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:34,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:35,579 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:11:36,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:37,364 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:37,365 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:37,365 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:37,370 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:38,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:38,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:40,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:11:40,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:11:40,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:11:42,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:42,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:42,655 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:42,655 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:42,660 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:43,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:44,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:46,273 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:46,274 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:46,275 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:46,279 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:47,180 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:47,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:48,183 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:49,684 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:49,685 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:49,685 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:49,690 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:50,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:50,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:52,691 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:54,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:54,287 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:54,287 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:54,292 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:55,188 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:11:55,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:11:55,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:11:55,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:11:56,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:11:58,180 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:58,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:58,275 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:11:58,275 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:58,280 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:59,194 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:00,196 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:02,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:02,014 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:02,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:02,019 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:02,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:04,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:04,200 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:05,581 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:12:05,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:05,741 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:05,742 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:05,747 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:06,204 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:06,204 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:09,747 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:10,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:10,830 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:10,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:10,835 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:10,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:12:10,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:12:10,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:12:11,207 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:12,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:15,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:15,747 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:15,748 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:15,748 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:15,753 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:16,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:16,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:20,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:21,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:21,019 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:21,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:21,024 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:21,217 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:22,219 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:25,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:12:25,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:12:25,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:12:26,170 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:28,127 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:28,128 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:28,128 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:28,133 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:28,222 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:30,226 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:32,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:32,115 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:32,115 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:32,115 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:32,120 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:32,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:34,230 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:35,587 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:12:37,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:37,045 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:37,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:37,050 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:37,232 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:38,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:38,234 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:40,722 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:40,723 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:40,723 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:40,728 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:40,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:12:40,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:12:40,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:12:41,237 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:42,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:43,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:44,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:44,442 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:44,442 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:44,446 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:45,241 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:46,243 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:48,486 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:48,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:48,768 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:48,768 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:48,772 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:49,245 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:50,248 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:52,743 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:52,744 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:52,744 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:52,749 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:53,250 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:53,749 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:54,252 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:55,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:12:55,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:12:55,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:12:56,789 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:56,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:12:56,789 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:56,794 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:57,254 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:12:58,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:12:58,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:01,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:01,026 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:01,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:01,031 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:01,259 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:02,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:04,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:04,796 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:04,797 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:04,797 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:04,802 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:05,263 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:05,585 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:13:06,266 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:08,764 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:08,765 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:08,765 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:08,770 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:09,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:09,770 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:10,270 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:10,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:13:10,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:13:10,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:13:13,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:13,674 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:13,674 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:13,679 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:14,274 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:14,275 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:15,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:19,362 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:19,363 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:19,363 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:19,368 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:20,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:20,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:21,369 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:24,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:24,284 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:24,284 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:24,289 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:25,283 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:25,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:13:25,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:13:25,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:13:26,285 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:27,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:27,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:27,576 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:27,576 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:27,580 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:28,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:28,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:31,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:31,654 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:31,654 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:31,659 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:32,292 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:32,292 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:32,659 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:35,590 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:13:35,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:35,778 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:35,778 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:35,783 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:36,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:36,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:37,783 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:39,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:39,832 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:39,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:39,837 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:40,301 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:40,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:40,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:13:40,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:13:40,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:13:43,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:43,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:43,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:43,808 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:43,813 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:44,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:44,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:48,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:50,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:50,491 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:50,492 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:50,496 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:51,309 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:52,312 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:54,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:55,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:55,833 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:55,834 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:55,838 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:55,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:13:55,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:13:55,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:13:56,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:13:56,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:13:59,780 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:59,780 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:13:59,781 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:59,781 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:59,786 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:00,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:00,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:04,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:05,589 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:14:08,342 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:08,342 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:08,343 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:08,348 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:09,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:10,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:10,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:10,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:14:10,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:14:10,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:14:12,460 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:12,460 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:12,461 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:12,466 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:13,332 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:14,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:15,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:16,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:16,677 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:16,678 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:16,683 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:17,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:18,338 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:20,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:20,791 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:20,792 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:20,792 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:20,797 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:21,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:22,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:25,273 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:25,274 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:25,274 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:25,279 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:25,344 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:25,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:14:25,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:14:25,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:14:26,145 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:26,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:29,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:29,063 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:29,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:29,068 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:29,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:30,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:32,105 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:34,447 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:34,448 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:34,449 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:34,453 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:35,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:35,597 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:14:36,356 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:37,599 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:39,906 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:39,907 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:39,907 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:39,912 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:40,361 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:40,361 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:40,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:14:40,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:14:40,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:14:43,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:44,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:44,057 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:44,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:44,062 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:44,365 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:44,365 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:49,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:49,124 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:49,125 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:49,125 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:49,130 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:49,368 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:50,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:54,131 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:55,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:14:55,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:14:55,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:14:57,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:57,294 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:14:57,294 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:57,299 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:57,374 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:14:58,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:14:59,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:02,489 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:02,490 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:02,491 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:02,495 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:03,380 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:04,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:04,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:05,594 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:15:06,890 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:06,890 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:06,891 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:06,895 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:07,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:08,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:09,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:10,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:15:10,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:15:10,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:15:12,915 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:12,916 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:12,916 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:12,921 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:13,395 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:14,397 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:14,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:17,360 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:17,361 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:17,361 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:17,366 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:17,401 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:18,403 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:20,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:22,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:22,041 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:22,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:22,046 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:22,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:22,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:25,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:15:25,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:15:25,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:15:26,198 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:26,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:26,428 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:26,428 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:26,433 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:27,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:28,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:31,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:33,763 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:33,764 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:33,764 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:33,769 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:34,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:34,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:35,596 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:15:36,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:39,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:39,150 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:39,151 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:39,156 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:39,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:40,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:40,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:15:40,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:15:40,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:15:42,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:43,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:43,729 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:43,730 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:43,734 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:44,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:44,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:47,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:47,440 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:47,440 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:47,440 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:47,445 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:47,445 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:48,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:52,447 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:54,108 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:54,109 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:54,110 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:54,114 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:54,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:54,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:15:55,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:15:55,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:15:55,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:15:58,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:58,013 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:15:58,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:58,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:58,018 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:58,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:15:58,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:02,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:02,777 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:02,778 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:02,783 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:03,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:03,783 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:04,469 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:05,598 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:16:08,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:08,704 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:08,704 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:08,709 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:09,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:09,709 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:10,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:10,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:16:10,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:16:10,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:16:12,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:12,994 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:12,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:13,000 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:13,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:14,480 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:15,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:17,780 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:17,780 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:17,781 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:17,789 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:18,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:18,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:20,789 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:22,131 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:22,132 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:22,132 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:22,137 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:22,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:22,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:25,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:16:25,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:16:25,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:16:26,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:30,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:30,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:30,695 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:30,700 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:31,493 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:31,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:32,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:34,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:34,428 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:34,428 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:34,433 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:34,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:35,600 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:16:36,500 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:37,601 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:40,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:16:40,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:16:40,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:16:41,342 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:41,343 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:41,343 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:41,348 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:41,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:42,507 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:43,386 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:45,125 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:45,126 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:45,127 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:45,131 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:45,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:46,511 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:49,132 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:49,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:49,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:49,278 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:49,283 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:49,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:50,515 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:54,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:54,048 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:54,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:54,054 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:54,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:54,520 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:16:55,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:55,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:16:55,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:16:55,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:16:57,892 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:57,893 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:16:57,893 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:57,898 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:58,525 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:16:58,525 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:00,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:04,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:04,213 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:04,213 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:04,218 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:04,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:04,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:05,605 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:17:06,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:08,180 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:08,180 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:08,181 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:08,186 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:08,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:08,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:10,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:17:10,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:17:10,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:17:12,118 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:12,118 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:12,119 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:12,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:12,124 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:12,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:12,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:16,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:16,138 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:16,138 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:16,143 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:16,544 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:16,544 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:17,143 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:20,471 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:20,472 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:20,472 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:20,477 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:20,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:22,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:22,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:25,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:17:25,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:17:25,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:17:27,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:27,090 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:27,090 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:27,095 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:27,552 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:28,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:28,555 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:30,404 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:30,405 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:30,405 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:30,410 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:30,556 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:32,559 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:33,410 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:34,878 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:34,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:34,879 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:34,884 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:35,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:35,604 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:17:36,563 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:38,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:40,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:17:40,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:17:40,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:17:41,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:41,703 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:41,703 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:41,708 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:42,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:42,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:43,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:45,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:45,642 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:45,642 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:45,647 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:46,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:46,574 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:49,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:50,487 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:50,488 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:50,488 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:50,493 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:50,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:52,579 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:17:54,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:54,691 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:17:54,691 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:54,691 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:54,696 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:55,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:17:55,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:17:55,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:17:55,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:17:56,583 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:00,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:00,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:00,580 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:00,581 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:00,585 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:00,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:02,590 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:05,609 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:18:05,612 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:06,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:06,024 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:06,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:06,029 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:06,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:06,595 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:10,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:18:10,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:18:10,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:18:11,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:13,743 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:13,744 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:13,744 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:13,749 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:14,601 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:14,601 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:16,785 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:17,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:17,553 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:17,554 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:17,559 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:17,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:18,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:21,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:21,997 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:21,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:21,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:22,002 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:22,610 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:22,610 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:25,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:18:25,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:18:25,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:18:26,431 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:26,432 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:26,432 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:26,437 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:26,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:27,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:28,615 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:29,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:29,959 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:29,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:29,964 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:30,618 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:30,618 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:32,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:34,878 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:34,879 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:34,879 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:34,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:35,608 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:18:35,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:36,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:38,610 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:40,506 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:40,507 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:40,507 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:40,512 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:40,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:40,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:18:40,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:18:40,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:18:42,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:44,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:44,029 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:44,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:44,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:44,034 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:44,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:44,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:48,876 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:48,877 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:48,877 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:48,882 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:49,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:49,883 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:50,638 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:52,927 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:52,928 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:52,928 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:52,933 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:53,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:54,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:18:54,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:55,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:18:55,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:18:55,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:18:56,313 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:56,313 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:18:56,313 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:56,318 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:56,645 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:18:56,645 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:00,358 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:01,583 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:01,584 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:01,584 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:01,589 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:01,649 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:02,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:05,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:05,610 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:19:08,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:08,290 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:08,290 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:08,295 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:08,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:08,658 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:10,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:19:10,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:19:10,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:19:11,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:11,817 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:11,817 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:11,818 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:11,822 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:12,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:12,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:16,862 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:18,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:18,742 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:18,742 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:18,747 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:19,666 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:20,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:22,747 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:23,405 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:23,405 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:23,406 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:23,410 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:23,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:24,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:25,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:19:25,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:19:25,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:19:27,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:27,968 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:27,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:27,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:27,973 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:28,677 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:28,677 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:32,127 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:32,128 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:32,128 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:32,133 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:32,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:32,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:33,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:35,612 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:19:36,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:36,255 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:36,255 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:36,261 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:36,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:36,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:38,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:39,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:39,924 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:39,925 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:39,930 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:40,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:40,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:40,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:19:40,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:19:40,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:19:44,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:44,471 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:44,471 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:44,471 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:44,476 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:44,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:46,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:48,718 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:48,719 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:48,719 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:48,724 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:49,697 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:49,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:50,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:52,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:52,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:52,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:52,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:52,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:54,706 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:19:55,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:55,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:19:55,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:19:55,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:19:56,487 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:56,488 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:19:56,488 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:56,492 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:56,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:19:58,711 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:00,511 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:00,512 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:00,512 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:00,517 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:00,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:01,518 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:02,715 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:04,555 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:04,556 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:04,556 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:04,561 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:04,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:05,615 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:20:06,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:06,720 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:08,638 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:08,639 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:08,639 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:08,644 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:08,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:10,724 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:10,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:20:10,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:20:10,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:20:12,204 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:12,594 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:12,595 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:12,595 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:12,600 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:12,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:14,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:16,595 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:16,595 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:16,596 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:16,600 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:16,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:17,601 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:18,732 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:22,601 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:23,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:23,504 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:23,504 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:23,509 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:23,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:24,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:25,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:20:25,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:20:25,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:20:27,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:27,995 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:27,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:27,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:28,001 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:28,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:28,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:32,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:32,936 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:32,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:32,941 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:33,745 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:33,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:34,747 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:35,620 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:20:36,764 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:36,765 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:36,765 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:36,770 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:37,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:38,751 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:39,770 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:40,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:20:40,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:20:40,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:20:42,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:42,154 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:42,155 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:42,160 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:42,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:42,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:45,198 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:47,846 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:47,847 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:47,847 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:47,852 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:48,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:48,762 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:20:50,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:55,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:55,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:20:55,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:20:55,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:20:56,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:56,267 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:20:56,267 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:56,272 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:56,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:20:56,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:00,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:00,143 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:00,144 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:00,148 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:00,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:00,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:01,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:04,804 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:04,804 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:04,805 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:04,809 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:05,619 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:21:05,776 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:06,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:06,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:09,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:09,010 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:09,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:09,015 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:09,780 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:10,783 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:10,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:21:10,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:21:10,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:21:12,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:15,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:15,022 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:15,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:15,027 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:15,786 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:16,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:18,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:21,187 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:21,188 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:21,188 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:21,192 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:21,791 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:22,793 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:23,193 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:25,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:21:25,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:21:25,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:21:26,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:26,193 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:26,193 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:26,197 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:26,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:26,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:28,198 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:30,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:30,225 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:30,226 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:30,230 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:30,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:30,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:33,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:35,624 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:21:36,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:36,246 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:36,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:36,251 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:36,807 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:36,807 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:38,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:40,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:21:40,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:21:40,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:21:42,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:42,010 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:42,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:42,015 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:42,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:42,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:44,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:46,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:46,052 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:46,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:46,057 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:46,817 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:46,817 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:49,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:50,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:50,404 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:50,404 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:50,409 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:50,821 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:50,821 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:54,410 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:54,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:54,658 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:54,658 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:54,663 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:54,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:21:55,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:21:55,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:21:55,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:21:56,826 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:21:59,446 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:59,447 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:21:59,448 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:59,448 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:59,452 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:59,830 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:00,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:03,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:03,315 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:03,315 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:03,320 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:03,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:04,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:05,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:05,623 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:22:10,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:10,505 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:10,505 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:10,506 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:10,510 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:10,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:10,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:10,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:22:10,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:22:10,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:22:14,752 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:14,752 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:14,753 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:14,757 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:14,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:15,794 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:16,847 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:20,794 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:21,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:21,262 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:21,263 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:21,268 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:21,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:22,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:25,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:25,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:25,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:25,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:25,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:25,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:22:25,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:22:25,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:22:26,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:26,856 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:29,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:29,718 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:29,718 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:29,723 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:29,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:30,860 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:31,761 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:33,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:33,259 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:33,259 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:33,264 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:33,862 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:34,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:35,625 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:22:37,095 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:37,096 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:37,096 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:37,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:37,101 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:37,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:38,869 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:40,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:22:40,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:22:40,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:22:42,203 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:42,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:42,555 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:42,555 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:42,560 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:42,874 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:42,874 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:47,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:47,590 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:47,590 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:47,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:47,595 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:47,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:48,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:51,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:51,491 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:51,491 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:51,496 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:51,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:52,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:53,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:55,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:22:55,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:22:55,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:22:56,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:56,945 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:22:56,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:56,950 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:57,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:22:58,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:22:58,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:01,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:01,320 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:01,320 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:01,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:01,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:02,895 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:04,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:04,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:04,859 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:04,859 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:04,864 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:04,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:05,627 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:23:06,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:09,182 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:09,183 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:09,183 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:09,189 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:09,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:10,189 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:10,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:10,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:23:10,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:23:10,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:23:13,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:13,028 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:13,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:13,033 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:13,906 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:14,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:16,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:16,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:16,796 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:16,796 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:16,801 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:16,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:18,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:20,760 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:20,761 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:20,761 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:20,766 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:20,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:21,766 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:22,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:24,735 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:24,736 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:24,736 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:24,741 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:24,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:25,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:23:25,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:23:25,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:23:26,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:27,143 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:29,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:29,939 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:29,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:29,944 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:30,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:30,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:32,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:35,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:35,028 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:35,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:35,034 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:35,629 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:23:35,926 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:36,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:38,631 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:40,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:23:40,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:23:40,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:23:43,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:43,995 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:43,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:43,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:44,000 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:44,936 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:44,936 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:48,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:48,930 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:48,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:48,935 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:48,938 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:49,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:50,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:53,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:53,622 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:53,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:53,627 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:53,943 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:54,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:23:55,627 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:55,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:23:55,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:23:55,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:23:57,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:57,442 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:23:57,442 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:57,446 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:57,947 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:23:58,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:01,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:01,385 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:01,386 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:01,386 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:01,390 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:01,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:02,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:04,978 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:04,978 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:04,979 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:04,983 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:05,631 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:24:05,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:06,632 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:06,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:09,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:09,298 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:09,298 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:09,303 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:09,961 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:10,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:24:10,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:24:10,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:24:10,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:12,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:13,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:13,157 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:13,157 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:13,162 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:13,966 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:14,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:18,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:19,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:19,047 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:19,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:19,053 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:19,972 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:20,974 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:24,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:24,732 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:24,733 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:24,734 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:24,738 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:24,978 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:24,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:25,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:24:25,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:24:25,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:24:28,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:28,504 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:28,504 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:28,508 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:28,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:28,983 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:29,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:32,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:32,679 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:32,680 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:32,684 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:32,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:32,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:34,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:35,633 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:24:38,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:38,393 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:38,393 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:38,398 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:38,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:38,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:40,399 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:40,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:24:40,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:24:40,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:24:46,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:48,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:48,428 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:48,428 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:48,433 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:49,000 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:49,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:51,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:52,356 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:52,356 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:24:52,357 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:52,362 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:53,005 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:24:53,005 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:24:55,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:24:55,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:24:55,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:24:57,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:00,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:00,022 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:00,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:00,029 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:01,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:25:01,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:25:03,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:05,372 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:05,373 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:05,373 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:05,378 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:05,639 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:25:06,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:25:07,018 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:25:08,640 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:10,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:25:10,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:25:10,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:25:13,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:13,798 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:13,798 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:13,798 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:13,803 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:14,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:25:15,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:25:18,841 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:19,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:19,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:19,699 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:19,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:20,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:25:21,030 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:25:24,098 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:24,098 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:24,099 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:24,099 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:24,103 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:25,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:25:25,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:25:25,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:25:25,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:25:25,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:25:29,146 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:30,071 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:30,071 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:30,071 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:30,076 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:31,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:25:31,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:25:33,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:33,617 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:33,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:33,622 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:34,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:25:34,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:35,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:25:35,638 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:25:39,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:39,222 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:39,222 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:39,227 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:40,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:25:40,227 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:40,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:25:40,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:25:40,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:25:41,047 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:25:45,232 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:45,500 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:45,500 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:45,501 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:45,505 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:46,050 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:25:47,053 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:25:50,542 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:53,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:53,986 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:53,986 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:53,991 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:54,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:25:55,059 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:25:55,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:25:55,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:25:55,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:25:56,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:59,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:59,367 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:25:59,367 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:59,371 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:00,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:01,065 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:01,410 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:04,789 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:04,790 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:04,790 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:04,795 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:05,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:05,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:05,643 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:26:06,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:10,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:26:10,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:26:10,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:26:12,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:12,377 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:12,378 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:12,378 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:12,383 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:13,078 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:13,078 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:16,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:16,926 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:16,926 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:16,931 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:17,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:17,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:19,084 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:21,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:21,608 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:21,609 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:21,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:22,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:23,088 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:23,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:25,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:25,702 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:25,702 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:25,707 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:25,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:26:25,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:26:25,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:26:26,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:27,093 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:29,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:30,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:30,998 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:30,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:31,003 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:31,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:33,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:35,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:35,131 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:35,132 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:35,132 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:35,138 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:35,642 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:26:36,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:37,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:38,860 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:38,861 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:38,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:38,866 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:39,106 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:39,106 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:40,867 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:40,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:26:40,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:26:40,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:26:44,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:44,961 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:44,961 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:44,966 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:45,111 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:46,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:47,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:51,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:51,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:51,699 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:51,699 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:51,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:52,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:53,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:26:55,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:26:55,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:26:55,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:26:56,140 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:56,169 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:26:56,169 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:56,169 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:56,176 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:57,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:26:57,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:00,735 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:00,735 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:00,736 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:00,740 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:01,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:01,129 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:01,778 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:04,462 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:04,463 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:04,463 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:04,475 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:05,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:05,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:05,644 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:27:07,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:08,906 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:08,907 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:08,908 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:08,912 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:09,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:09,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:10,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:27:10,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:27:10,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:27:13,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:14,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:14,276 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:14,277 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:14,282 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:15,143 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:15,143 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:18,282 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:18,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:18,914 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:18,915 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:18,919 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:19,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:19,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:23,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:23,744 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:23,745 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:23,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:23,750 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:24,150 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:25,153 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:25,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:27:25,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:27:25,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:27:27,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:27,917 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:27,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:27,923 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:28,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:28,923 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:29,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:33,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:34,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:34,768 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:34,768 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:34,773 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:35,162 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:35,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:35,646 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:27:39,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:40,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:27:40,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:27:40,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:27:42,127 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:42,127 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:42,128 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:42,133 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:42,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:43,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:45,134 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:45,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:45,701 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:45,702 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:45,706 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:46,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:47,175 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:50,707 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:51,714 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:51,715 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:51,715 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:51,720 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:52,177 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:53,180 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:27:55,721 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:55,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:55,874 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:27:55,874 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:55,879 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:55,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:27:55,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:27:55,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:27:56,182 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:27:57,184 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:00,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:00,939 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:00,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:00,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:00,944 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:01,190 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:01,190 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:05,110 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:05,111 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:05,111 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:05,116 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:05,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:05,648 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:28:06,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:07,196 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:09,531 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:09,531 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:09,532 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:09,537 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:10,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:10,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:28:10,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:28:10,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:28:11,200 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:12,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:14,977 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:14,978 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:14,978 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:14,983 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:15,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:17,205 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:18,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:19,393 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:19,394 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:19,394 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:19,399 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:20,207 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:21,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:23,400 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:23,801 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:23,802 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:23,802 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:23,807 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:24,211 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:25,213 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:25,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:28:25,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:28:25,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:28:29,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:32,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:32,014 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:32,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:32,020 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:32,217 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:33,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:35,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:35,653 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:28:37,691 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:37,692 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:37,692 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:37,697 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:38,223 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:39,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:40,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:40,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:28:40,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:28:40,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:28:41,652 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:41,652 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:41,653 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:41,658 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:42,228 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:43,230 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:46,429 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:46,430 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:46,430 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:46,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:46,436 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:47,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:47,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:51,436 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:54,850 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:54,851 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:54,851 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:54,856 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:55,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:28:55,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:28:55,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:28:55,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:28:57,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:57,243 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:28:58,781 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:58,782 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:28:58,782 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:58,788 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:59,245 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:01,248 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:02,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:02,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:02,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:02,790 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:02,795 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:03,250 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:05,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:05,652 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:29:08,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:10,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:29:10,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:29:10,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:29:11,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:11,415 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:11,415 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:11,421 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:12,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:13,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:14,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:16,494 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:16,495 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:16,495 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:16,500 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:17,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:17,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:19,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:24,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:24,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:24,533 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:24,533 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:24,538 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:25,271 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:25,271 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:25,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:29:25,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:29:25,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:29:28,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:28,742 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:28,742 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:28,747 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:29,273 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:29,785 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:31,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:32,905 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:32,906 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:32,906 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:32,911 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:33,277 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:34,912 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:35,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:35,654 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:29:37,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:37,085 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:37,086 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:37,091 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:37,281 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:39,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:40,091 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:40,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:29:40,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:29:40,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:29:42,763 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:42,767 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:42,767 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:42,773 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:43,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:45,291 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:45,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:46,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:46,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:46,808 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:46,813 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:47,292 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:49,295 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:50,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:50,920 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:50,920 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:50,921 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:50,925 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:51,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:53,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:29:54,668 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:54,669 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:29:54,669 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:54,674 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:55,301 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:29:55,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:29:55,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:29:55,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:29:56,169 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:57,304 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:00,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:00,264 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:00,265 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:00,270 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:00,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:01,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:01,309 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:05,656 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:30:05,660 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:05,661 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:05,661 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:05,666 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:06,311 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:06,666 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:07,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:10,500 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:10,501 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:10,501 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:10,506 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:10,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:30:10,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:30:10,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:30:11,318 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:11,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:12,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:14,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:14,859 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:14,859 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:14,864 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:15,321 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:17,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:17,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:22,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:23,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:23,172 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:23,172 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:23,177 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:23,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:25,330 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:25,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:30:25,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:30:25,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:30:26,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:26,806 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:26,807 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:26,812 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:27,332 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:28,812 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:29,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:33,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:34,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:34,131 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:34,131 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:34,136 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:34,337 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:35,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:35,658 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:30:37,891 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:37,892 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:37,892 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:37,897 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:38,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:38,897 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:39,344 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:40,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:30:40,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:30:40,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:30:43,424 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:43,424 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:43,425 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:43,430 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:44,349 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:44,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:45,351 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:48,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:48,317 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:48,317 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:48,322 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:48,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:49,356 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:50,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:52,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:52,159 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:52,159 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:52,164 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:52,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:53,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:55,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:55,340 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:55,340 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:55,340 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:55,345 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:55,361 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:30:55,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:30:55,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:30:55,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:30:57,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:30:59,850 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:59,851 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:30:59,852 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:59,856 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:00,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:00,857 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:01,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:05,664 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:31:06,665 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:08,448 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:08,449 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:08,449 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:08,454 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:09,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:09,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:10,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:31:10,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:31:10,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:31:12,168 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:12,895 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:12,896 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:12,896 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:12,901 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:13,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:15,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:16,987 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:16,988 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:16,988 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:16,993 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:17,384 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:17,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:19,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:21,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:21,811 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:21,811 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:21,816 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:22,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:23,391 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:23,817 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:25,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:31:25,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:31:25,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:31:28,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:28,245 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:28,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:28,250 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:28,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:29,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:29,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:34,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:34,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:34,730 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:34,731 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:34,736 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:35,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:35,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:35,663 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:31:39,664 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:39,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:39,957 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:39,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:39,962 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:40,405 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:40,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:31:40,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:31:40,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:31:41,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:45,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:45,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:45,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:45,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:45,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:46,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:47,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:50,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:51,487 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:51,488 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:51,488 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:51,493 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:52,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:53,420 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:55,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:31:55,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:31:55,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:31:56,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:56,164 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:56,165 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:56,165 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:56,170 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:56,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:31:57,424 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:31:59,816 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:59,817 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:31:59,817 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:59,822 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:00,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:32:01,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:32:01,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:03,911 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:03,912 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:32:03,912 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:03,917 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:04,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:32:05,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:32:05,667 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:32:07,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:10,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:32:10,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:32:10,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:32:11,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:11,266 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:32:11,266 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:11,271 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:11,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:32:13,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:13,442 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:32:18,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:18,010 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:32:18,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:18,015 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:18,445 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:32:19,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:19,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:32:23,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:23,070 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:32:23,070 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:23,075 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:23,449 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:32:24,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:25,452 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:32:25,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:32:25,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:32:25,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:32:29,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:29,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:29,347 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:32:29,347 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:29,352 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:29,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:32:31,458 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:32:33,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:33,390 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:32:33,390 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:33,395 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:33,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:32:34,396 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:35,463 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:32:35,667 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:32:38,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:38,747 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:32:38,747 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:38,752 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:39,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:32:39,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:32:39,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:40,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:32:40,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:32:40,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:32:45,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:47,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:47,867 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:32:47,867 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:47,872 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:48,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:32:49,475 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:32:50,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:52,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:52,233 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:32:52,234 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:52,238 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:52,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:32:53,479 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:32:55,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:32:55,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:32:55,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:32:56,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:00,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:00,336 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:00,336 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:00,341 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:00,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:01,342 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:01,487 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:33:05,669 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:33:06,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:10,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:10,821 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:10,821 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:10,826 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:10,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:33:10,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:33:10,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:33:11,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:11,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:33:12,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:16,869 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:16,875 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:16,875 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:16,880 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:17,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:17,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:19,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:33:22,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:23,495 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:23,496 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:23,496 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:23,501 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:23,505 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:25,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:33:25,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:33:25,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:33:25,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:33:27,345 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:27,346 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:27,346 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:27,351 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:27,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:28,351 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:29,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:33:33,352 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:34,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:34,490 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:34,490 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:34,495 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:34,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:35,518 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:33:35,671 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:33:38,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:38,409 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:38,409 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:38,409 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:38,414 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:38,520 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:39,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:33:40,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:33:40,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:33:40,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:33:42,288 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:42,289 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:42,289 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:42,294 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:42,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:43,528 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:33:44,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:49,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:49,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:49,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:49,697 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:49,702 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:50,532 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:51,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:33:53,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:53,704 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:53,705 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:53,709 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:54,536 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:54,710 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:55,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:33:55,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:33:55,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:33:55,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:33:58,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:58,032 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:33:58,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:58,037 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:58,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:33:59,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:00,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:03,387 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:03,388 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:03,388 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:03,393 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:03,547 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:05,393 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:05,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:05,673 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:34:07,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:07,571 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:07,571 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:07,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:08,552 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:09,555 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:10,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:10,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:34:10,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:34:10,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:34:13,457 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:13,458 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:13,458 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:13,463 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:13,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:15,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:16,464 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:17,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:17,316 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:17,317 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:17,322 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:17,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:19,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:21,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:21,718 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:21,718 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:21,719 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:21,723 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:22,567 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:23,570 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:25,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:34:25,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:34:25,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:34:26,168 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:26,192 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:26,192 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:26,197 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:26,572 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:27,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:27,574 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:31,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:31,064 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:31,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:31,069 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:31,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:33,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:33,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:35,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:35,469 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:35,469 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:35,474 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:35,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:35,678 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:34:37,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:38,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:40,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:34:40,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:34:40,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:34:42,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:42,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:42,523 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:42,527 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:42,588 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:43,590 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:44,528 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:46,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:46,986 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:46,986 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:46,991 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:47,593 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:49,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:49,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:52,883 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:52,884 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:52,884 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:52,890 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:53,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:53,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:34:55,890 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:55,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:34:55,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:34:55,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:34:56,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:56,823 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:34:56,823 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:56,828 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:57,604 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:34:57,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:01,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:02,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:02,032 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:02,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:02,037 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:02,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:03,611 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:05,677 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:35:07,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:08,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:08,626 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:08,626 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:08,631 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:09,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:09,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:10,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:35:10,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:35:10,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:35:12,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:12,329 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:12,329 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:12,334 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:12,619 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:13,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:13,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:18,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:18,766 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:18,767 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:18,767 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:18,773 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:19,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:19,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:23,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:23,213 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:23,213 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:23,218 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:23,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:24,219 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:25,634 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:25,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:35:25,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:35:25,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:35:27,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:27,196 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:27,196 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:27,201 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:27,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:29,638 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:30,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:31,099 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:31,100 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:31,100 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:31,105 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:31,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:33,643 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:34,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:34,671 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:34,671 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:34,676 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:35,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:35,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:35,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:35,682 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:35:38,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:38,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:38,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:38,996 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:39,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:39,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:40,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:35:40,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:35:40,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:35:41,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:43,220 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:43,221 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:43,221 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:43,226 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:43,654 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:45,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:46,227 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:47,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:47,585 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:47,585 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:47,591 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:47,658 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:49,661 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:51,583 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:51,584 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:51,584 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:51,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:51,589 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:51,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:53,666 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:35:55,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:35:55,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:35:55,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:35:55,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:56,215 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:35:56,215 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:56,220 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:56,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:35:57,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:57,670 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:02,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:04,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:04,382 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:04,382 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:04,387 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:04,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:05,678 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:05,681 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:36:07,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:08,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:08,301 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:08,301 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:08,306 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:08,680 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:09,682 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:10,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:36:10,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:36:10,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:36:13,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:13,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:13,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:13,512 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:13,517 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:13,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:15,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:18,518 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:18,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:18,925 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:18,925 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:18,930 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:19,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:19,693 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:23,425 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:23,426 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:23,426 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:23,431 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:23,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:24,431 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:25,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:25,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:36:25,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:36:25,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:36:27,531 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:27,532 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:27,532 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:27,537 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:27,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:29,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:29,702 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:31,708 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:31,709 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:31,709 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:31,714 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:32,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:33,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:34,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:35,683 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:36:36,565 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:36,565 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:36,566 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:36,571 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:36,708 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:37,711 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:40,572 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:40,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:40,810 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:40,810 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:40,815 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:40,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:36:40,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:36:40,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:36:41,716 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:41,716 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:44,755 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:44,756 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:44,756 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:44,761 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:45,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:45,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:45,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:50,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:52,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:52,958 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:52,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:52,963 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:53,727 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:36:53,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:36:55,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:36:55,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:36:55,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:36:56,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:59,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:59,323 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:36:59,324 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:59,330 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:59,733 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:01,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:01,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:03,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:03,630 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:03,630 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:03,635 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:03,737 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:05,686 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:37:05,740 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:06,687 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:10,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:10,384 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:10,385 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:10,390 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:10,744 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:10,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:37:10,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:37:10,939 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:37:11,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:12,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:14,844 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:14,845 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:14,845 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:14,850 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:15,751 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:15,751 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:17,851 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:22,851 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:23,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:23,318 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:23,318 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:23,323 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:23,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:25,759 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:25,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:37:25,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:37:25,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:37:28,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:28,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:28,382 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:28,382 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:28,388 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:28,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:29,763 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:33,178 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:33,179 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:33,179 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:33,179 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:33,184 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:33,765 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:35,688 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:37:35,769 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:36,983 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:36,983 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:36,984 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:36,989 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:37,772 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:37,772 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:38,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:40,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:37:40,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:37:40,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:37:42,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:42,364 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:42,364 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:42,369 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:42,776 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:43,779 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:44,370 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:47,357 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:47,358 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:47,359 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:47,364 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:47,781 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:49,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:50,365 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:53,299 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:53,300 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:53,300 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:53,305 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:53,786 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:55,789 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:37:55,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:37:55,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:37:55,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:37:56,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:58,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:58,590 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:37:58,591 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:58,596 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:58,791 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:37:59,795 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:01,596 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:04,118 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:04,119 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:04,119 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:04,124 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:04,798 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:05,693 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:38:05,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:06,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:08,721 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:08,722 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:08,722 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:08,727 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:08,803 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:09,806 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:10,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:38:10,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:38:10,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:38:12,188 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:14,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:14,166 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:14,167 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:14,171 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:14,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:15,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:18,088 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:18,089 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:18,089 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:18,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:18,094 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:18,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:19,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:21,665 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:21,666 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:21,666 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:21,671 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:21,817 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:23,671 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:23,820 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:25,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:25,307 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:25,307 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:25,311 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:25,821 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:25,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:38:25,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:38:25,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:38:27,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:29,185 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:34,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:34,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:34,641 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:34,642 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:34,646 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:34,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:35,692 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:38:35,831 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:39,187 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:39,188 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:39,188 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:39,188 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:39,193 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:39,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:39,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:40,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:38:40,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:38:40,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:38:43,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:43,141 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:43,142 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:43,146 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:43,841 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:43,841 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:45,147 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:48,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:48,439 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:48,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:48,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:48,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:49,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:50,445 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:52,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:52,937 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:52,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:52,942 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:53,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:53,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:38:55,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:38:55,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:38:55,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:38:56,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:56,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:56,882 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:38:56,882 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:56,887 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:57,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:38:57,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:00,627 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:00,628 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:00,628 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:00,633 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:00,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:01,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:01,860 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:05,697 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:39:06,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:06,022 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:06,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:06,027 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:06,863 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:07,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:07,866 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:10,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:39:10,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:39:10,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:39:12,172 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:14,811 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:14,812 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:14,812 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:14,817 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:14,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:15,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:17,818 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:18,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:18,941 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:18,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:18,946 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:19,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:19,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:22,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:23,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:23,150 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:23,150 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:23,155 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:23,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:23,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:25,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:39:25,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:39:25,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:39:27,168 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:27,169 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:27,169 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:27,174 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:27,886 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:27,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:28,174 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:33,175 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:33,404 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:33,405 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:33,405 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:33,410 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:33,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:35,696 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:39:35,893 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:37,675 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:37,676 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:37,676 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:37,681 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:37,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:38,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:39,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:40,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:39:40,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:39:40,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:39:42,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:42,027 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:42,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:42,032 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:42,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:43,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:44,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:46,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:46,282 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:46,282 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:46,287 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:46,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:47,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:49,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:51,770 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:51,771 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:51,771 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:51,776 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:51,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:53,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:39:54,776 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:55,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:39:55,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:39:55,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:39:55,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:56,187 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:39:56,187 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:56,191 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:56,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:39:57,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:00,192 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:02,705 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:02,706 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:02,707 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:02,712 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:02,922 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:03,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:05,701 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:40:05,702 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:06,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:06,955 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:06,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:06,960 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:07,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:07,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:10,926 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:10,927 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:10,927 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:10,927 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:10,932 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:10,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:10,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:40:10,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:40:10,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:40:11,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:15,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:15,136 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:15,136 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:15,141 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:15,940 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:15,940 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:16,178 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:21,178 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:21,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:21,998 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:21,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:22,003 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:22,944 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:23,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:25,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:40:25,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:40:25,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:40:26,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:26,607 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:26,608 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:26,608 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:26,612 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:26,948 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:27,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:31,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:31,074 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:31,074 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:31,079 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:31,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:31,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:32,079 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:35,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:35,362 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:35,362 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:35,367 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:35,701 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:40:35,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:35,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:37,702 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:38,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:38,881 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:38,881 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:38,886 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:38,961 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:39,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:40,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:40:40,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:40:40,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:40:43,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:43,612 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:43,613 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:43,613 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:43,618 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:43,968 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:45,972 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:48,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:49,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:49,162 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:49,163 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:49,168 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:49,976 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:49,976 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:53,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:53,090 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:53,090 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:53,095 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:53,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:53,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:40:54,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:55,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:40:55,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:40:55,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:40:58,475 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:58,476 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:40:58,476 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:58,481 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:58,983 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:40:59,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:59,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:03,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:03,018 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:03,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:03,023 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:03,991 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:41:03,991 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:05,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:05,702 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:41:10,498 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:10,499 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:10,499 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:10,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:10,504 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:10,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:41:10,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:41:10,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:41:10,995 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:41:11,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:15,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:15,302 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:15,303 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:15,308 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:16,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:41:16,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:16,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:20,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:20,945 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:20,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:20,950 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:21,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:41:21,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:22,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:24,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:24,546 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:24,547 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:24,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:25,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:41:25,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:41:25,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:41:25,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:41:26,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:27,192 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:28,227 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:28,227 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:28,228 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:28,233 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:29,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:41:30,018 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:32,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:32,013 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:32,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:32,017 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:32,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:41:33,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:34,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:35,704 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:41:38,706 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:40,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:40,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:40,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:40,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:40,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:41:40,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:41:40,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:41:41,026 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:41:42,030 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:44,146 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:45,718 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:45,718 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:45,719 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:45,723 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:46,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:41:48,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:49,724 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:54,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:54,134 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:54,134 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:54,140 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:55,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:41:55,140 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:55,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:41:55,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:41:55,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:41:56,045 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:41:58,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:58,471 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:41:58,471 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:58,477 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:59,047 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:00,051 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:00,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:03,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:03,571 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:42:03,571 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:03,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:04,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:05,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:05,706 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:42:06,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:07,754 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:07,755 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:42:07,755 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:07,760 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:08,058 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:10,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:10,761 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:10,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:42:10,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:42:10,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:42:12,763 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:12,764 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:42:12,764 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:12,769 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:13,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:14,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:15,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:17,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:17,335 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:42:17,335 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:17,340 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:18,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:18,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:20,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:20,959 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:42:20,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:20,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:20,965 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:21,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:22,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:25,455 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:25,456 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:42:25,456 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:25,461 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:25,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:42:25,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:42:25,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:42:26,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:26,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:26,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:31,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:31,282 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:31,283 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:42:31,283 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:31,288 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:32,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:32,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:35,716 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:42:36,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:36,202 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:42:36,203 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:36,207 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:37,088 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:37,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:38,091 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:40,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:42:40,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:42:40,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:42:42,651 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:42,652 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:42:42,652 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:42,652 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:42,657 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:43,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:44,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:47,658 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:51,731 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:51,732 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:42:51,734 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:51,739 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:52,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:42:52,739 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:54,105 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:42:55,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:42:55,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:42:55,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:42:58,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:03,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:05,711 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:43:05,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:05,910 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:05,910 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:05,915 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:06,113 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:08,117 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:08,916 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:10,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:10,440 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:10,440 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:10,445 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:10,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:43:10,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:43:10,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:43:11,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:12,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:14,179 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:14,476 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:14,476 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:14,477 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:14,481 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:15,124 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:16,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:18,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:18,915 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:18,915 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:18,920 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:19,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:19,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:20,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:24,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:25,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:43:25,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:43:25,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:43:26,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:26,705 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:26,705 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:26,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:27,134 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:28,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:30,711 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:32,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:32,654 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:32,655 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:32,659 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:33,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:34,142 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:35,713 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:43:35,714 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:36,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:36,330 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:36,330 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:36,335 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:37,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:38,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:40,907 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:40,908 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:40,908 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:40,908 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:40,913 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:40,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:43:40,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:43:40,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:43:41,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:42,152 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:44,643 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:44,644 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:44,644 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:44,649 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:45,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:46,156 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:46,649 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:48,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:48,654 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:48,654 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:48,659 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:49,158 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:50,161 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:51,659 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:53,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:53,563 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:53,563 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:53,567 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:54,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:43:54,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:43:55,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:43:55,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:43:55,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:43:57,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:57,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:57,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:43:57,990 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:57,994 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:58,167 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:00,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:02,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:02,167 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:02,167 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:02,173 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:02,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:03,174 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:04,177 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:05,715 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:44:07,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:07,157 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:07,157 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:07,162 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:07,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:08,181 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:09,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:10,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:44:10,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:44:10,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:44:13,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:13,010 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:13,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:13,015 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:13,184 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:14,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:15,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:17,480 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:17,481 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:17,481 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:17,485 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:18,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:18,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:20,486 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:21,789 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:21,790 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:21,790 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:21,795 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:22,194 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:24,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:25,796 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:25,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:44:25,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:44:25,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:44:26,573 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:26,574 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:26,574 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:26,579 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:27,199 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:28,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:31,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:35,717 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:44:36,087 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:36,088 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:36,088 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:36,093 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:36,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:37,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:38,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:40,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:40,535 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:40,535 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:40,539 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:40,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:44:40,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:44:40,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:44:41,211 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:42,215 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:42,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:46,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:46,384 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:46,384 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:46,389 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:47,217 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:47,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:48,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:51,585 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:51,586 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:51,586 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:51,591 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:52,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:52,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:52,592 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:55,596 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:55,597 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:55,597 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:55,602 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:55,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:44:55,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:44:55,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:44:56,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:44:56,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:44:58,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:59,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:59,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:44:59,696 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:59,700 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:00,232 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:02,236 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:03,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:04,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:04,502 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:04,502 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:04,507 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:05,237 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:05,719 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:45:06,240 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:09,494 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:09,495 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:09,495 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:09,495 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:09,500 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:10,244 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:10,244 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:10,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:45:10,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:45:10,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:45:15,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:16,351 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:16,352 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:16,352 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:16,357 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:17,249 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:18,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:20,358 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:21,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:21,422 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:21,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:21,427 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:22,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:22,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:25,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:25,383 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:25,383 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:25,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:25,388 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:25,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:45:25,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:45:25,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:45:26,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:26,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:28,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:28,739 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:28,740 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:28,745 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:29,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:30,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:30,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:32,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:32,825 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:32,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:32,831 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:33,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:34,270 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:35,721 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:45:36,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:37,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:37,827 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:37,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:37,832 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:38,272 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:40,275 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:40,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:45:40,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:45:40,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:45:41,395 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:41,395 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:41,396 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:41,401 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:42,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:42,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:42,402 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:46,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:46,090 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:46,090 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:46,095 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:46,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:48,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:48,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:52,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:52,550 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:52,550 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:52,555 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:53,289 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:53,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:54,291 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:55,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:45:55,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:45:55,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:45:57,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:57,535 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:45:57,535 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:57,539 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:58,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:45:58,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:45:59,540 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:01,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:01,257 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:01,258 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:01,262 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:01,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:02,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:05,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:05,727 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:46:06,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:06,681 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:06,681 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:06,686 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:07,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:08,308 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:10,455 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:10,456 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:10,456 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:10,456 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:10,461 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:10,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:46:10,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:46:10,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:46:11,310 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:12,313 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:14,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:14,969 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:14,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:14,974 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:15,315 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:15,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:16,318 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:18,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:18,941 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:18,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:18,947 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:19,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:20,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:21,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:22,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:22,835 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:22,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:22,840 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:23,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:24,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:25,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:46:25,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:46:25,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:46:26,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:26,963 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:26,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:26,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:26,968 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:27,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:28,331 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:31,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:32,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:32,054 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:32,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:32,060 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:32,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:34,337 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:35,726 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:46:36,299 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:36,300 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:36,300 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:36,305 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:36,338 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:37,305 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:38,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:39,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:39,684 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:39,684 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:39,689 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:40,345 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:40,345 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:40,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:46:40,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:46:40,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:46:43,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:45,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:45,322 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:45,322 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:45,327 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:45,349 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:46,351 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:48,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:51,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:51,407 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:51,407 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:51,412 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:52,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:52,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:46:53,412 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:55,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:46:55,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:46:55,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:46:57,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:57,806 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:46:57,806 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:57,811 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:58,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:46:58,811 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:00,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:03,811 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:05,730 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:47:06,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:06,301 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:06,301 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:06,305 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:06,368 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:08,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:09,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:09,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:09,827 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:09,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:09,833 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:10,373 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:10,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:47:10,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:47:10,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:47:12,376 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:13,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:13,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:13,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:13,996 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:14,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:14,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:16,380 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:17,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:17,247 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:17,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:17,252 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:17,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:18,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:20,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:22,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:22,040 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:22,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:22,045 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:22,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:24,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:25,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:25,873 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:25,873 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:25,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:25,878 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:25,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:47:25,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:47:25,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:47:26,390 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:28,393 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:29,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:29,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:29,809 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:29,813 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:30,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:30,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:31,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:33,604 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:33,605 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:33,605 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:33,610 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:34,400 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:34,401 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:35,730 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:47:37,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:37,582 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:37,583 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:37,583 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:37,588 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:38,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:38,405 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:40,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:47:40,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:47:40,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:47:42,864 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:42,865 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:42,865 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:42,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:42,870 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:43,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:44,411 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:47,871 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:48,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:48,115 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:48,115 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:48,120 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:48,413 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:50,416 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:52,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:52,582 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:52,582 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:52,587 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:53,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:53,588 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:54,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:55,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:47:55,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:47:55,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:47:56,666 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:56,667 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:47:56,667 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:56,672 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:57,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:47:58,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:47:58,672 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:00,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:00,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:00,696 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:00,700 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:01,428 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:02,431 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:03,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:05,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:05,338 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:05,338 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:05,343 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:05,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:05,732 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:48:06,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:08,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:09,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:09,233 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:09,234 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:09,238 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:09,437 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:10,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:10,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:48:10,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:48:10,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:48:13,792 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:13,796 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:13,796 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:13,796 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:13,801 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:14,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:14,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:18,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:20,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:20,562 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:20,562 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:20,567 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:21,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:22,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:24,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:25,916 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:25,917 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:25,917 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:25,922 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:25,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:48:25,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:48:25,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:48:26,453 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:28,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:30,152 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:31,507 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:31,508 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:31,508 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:31,513 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:32,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:32,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:35,513 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:35,734 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:48:35,918 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:35,919 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:35,919 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:35,924 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:36,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:38,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:40,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:40,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:48:40,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:48:40,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:48:41,800 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:41,801 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:41,801 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:41,806 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:42,471 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:42,471 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:46,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:47,191 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:47,192 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:47,192 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:47,197 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:47,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:48,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:52,198 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:54,400 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:54,401 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:48:54,401 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:54,406 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:54,480 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:48:55,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:48:55,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:48:55,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:48:56,483 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:48:58,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:03,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:03,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:03,316 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:03,316 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:03,321 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:03,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:04,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:05,736 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:49:07,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:07,557 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:07,557 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:07,562 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:08,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:08,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:08,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:10,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:49:10,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:49:10,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:49:13,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:13,230 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:13,230 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:13,235 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:13,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:14,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:14,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:19,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:19,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:19,577 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:19,577 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:19,582 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:20,505 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:20,505 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:23,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:23,591 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:23,591 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:23,596 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:24,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:24,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:24,596 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:25,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:49:25,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:49:25,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:49:29,164 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:29,165 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:29,165 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:29,171 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:29,512 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:30,172 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:30,515 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:32,452 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:32,453 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:32,453 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:32,458 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:32,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:34,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:35,459 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:35,741 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:49:37,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:37,021 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:37,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:37,026 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:37,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:38,524 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:40,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:40,798 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:40,798 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:40,798 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:40,803 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:40,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:49:40,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:49:40,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:49:41,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:42,529 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:44,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:44,608 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:44,608 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:44,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:45,531 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:46,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:46,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:48,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:48,498 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:48,498 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:48,503 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:48,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:50,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:52,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:52,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:52,626 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:52,627 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:52,632 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:53,540 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:49:54,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:49:55,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:49:55,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:49:55,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:49:58,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:58,074 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:49:58,074 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:58,074 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:58,079 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:58,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:00,549 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:02,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:02,064 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:02,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:02,069 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:02,551 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:04,070 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:04,554 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:05,740 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:50:06,603 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:06,604 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:06,604 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:06,609 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:07,555 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:08,558 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:09,609 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:10,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:50:10,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:50:10,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:50:12,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:12,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:12,462 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:12,467 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:12,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:14,564 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:15,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:17,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:17,012 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:17,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:17,017 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:17,566 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:18,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:21,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:21,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:21,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:21,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:21,939 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:22,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:22,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:25,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:50:25,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:50:25,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:50:26,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:26,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:26,748 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:26,749 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:26,753 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:27,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:28,579 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:31,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:31,118 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:31,118 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:31,123 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:31,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:32,123 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:32,583 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:35,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:35,043 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:35,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:35,048 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:35,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:35,745 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:50:36,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:37,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:40,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:50:40,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:50:40,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:50:43,152 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:43,153 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:43,153 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:43,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:43,159 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:43,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:44,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:48,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:48,388 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:48,389 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:48,389 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:48,394 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:48,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:50,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:52,188 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:52,189 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:52,190 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:52,194 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:52,601 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:53,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:54,604 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:55,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:50:55,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:50:55,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:50:56,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:56,950 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:50:56,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:56,955 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:57,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:50:58,608 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:50:58,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:01,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:01,187 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:01,187 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:01,193 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:01,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:02,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:04,193 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:05,744 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:51:07,457 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:07,458 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:07,458 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:07,463 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:07,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:08,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:09,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:10,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:51:10,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:51:10,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:51:11,173 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:11,233 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:11,233 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:11,238 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:11,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:12,624 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:14,519 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:14,520 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:14,521 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:14,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:14,526 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:14,625 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:16,629 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:19,526 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:23,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:23,040 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:23,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:23,045 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:23,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:24,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:25,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:25,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:51:25,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:51:25,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:51:27,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:27,832 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:27,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:27,837 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:28,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:28,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:30,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:33,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:33,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:33,789 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:33,794 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:34,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:34,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:35,752 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:51:36,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:37,628 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:37,629 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:37,629 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:37,633 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:37,647 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:38,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:40,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:51:40,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:51:40,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:51:42,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:42,143 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:42,143 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:42,143 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:42,148 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:42,653 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:44,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:47,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:48,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:48,583 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:48,583 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:48,588 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:48,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:50,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:52,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:53,663 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:53,664 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:53,664 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:53,669 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:54,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:51:54,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:51:55,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:51:55,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:51:55,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:51:58,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:58,378 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:58,379 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:51:58,379 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:58,384 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:58,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:00,674 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:03,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:05,749 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:52:06,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:06,821 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:06,821 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:06,826 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:07,678 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:08,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:08,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:10,672 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:10,673 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:10,673 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:10,678 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:10,682 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:10,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:52:10,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:52:10,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:52:12,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:14,178 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:14,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:14,206 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:14,206 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:14,211 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:14,686 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:16,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:18,796 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:18,797 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:18,797 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:18,802 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:19,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:19,803 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:20,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:22,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:22,555 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:22,555 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:22,560 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:22,696 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:24,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:25,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:25,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:52:25,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:52:25,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:52:27,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:27,295 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:27,295 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:27,299 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:27,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:28,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:31,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:35,751 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:52:35,891 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:35,892 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:35,892 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:35,897 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:36,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:36,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:36,897 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:39,664 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:39,665 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:39,665 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:39,670 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:39,711 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:40,715 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:40,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:52:40,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:52:40,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:52:42,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:43,739 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:43,740 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:43,741 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:43,746 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:44,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:44,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:47,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:52,747 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:52,895 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:52,896 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:52,897 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:52,902 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:53,726 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:54,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:52:55,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:52:55,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:52:55,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:52:57,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:57,509 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:52:57,509 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:57,515 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:57,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:52:58,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:58,733 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:01,491 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:01,492 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:01,492 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:01,496 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:01,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:02,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:04,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:05,752 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:53:06,700 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:06,701 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:06,701 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:06,706 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:06,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:08,744 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:09,707 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:10,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:53:10,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:53:10,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:53:11,923 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:11,924 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:11,924 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:11,928 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:12,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:12,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:14,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:18,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:18,231 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:18,231 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:18,236 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:18,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:20,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:20,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:22,660 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:22,661 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:22,662 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:22,667 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:22,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:24,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:25,667 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:25,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:53:25,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:53:25,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:53:27,192 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:27,193 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:27,194 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:27,198 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:27,762 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:28,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:31,199 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:32,709 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:32,710 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:32,710 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:32,715 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:32,767 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:34,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:35,754 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:53:36,756 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:40,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:53:40,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:53:40,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:53:41,354 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:41,354 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:41,355 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:41,359 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:41,774 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:42,360 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:42,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:46,386 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:46,387 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:46,387 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:46,392 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:46,780 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:47,393 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:48,783 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:51,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:51,434 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:51,434 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:51,439 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:51,785 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:52,440 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:52,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:53:55,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:53:55,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:53:55,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:53:56,409 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:56,410 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:53:56,410 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:56,415 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:56,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:53:58,415 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:58,793 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:01,145 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:01,146 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:01,146 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:01,151 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:01,796 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:02,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:04,152 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:05,760 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:54:09,761 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:09,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:09,815 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:09,815 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:09,820 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:10,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:10,806 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:10,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:54:10,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:54:10,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:54:15,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:17,514 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:17,514 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:17,515 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:17,520 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:17,810 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:18,812 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:20,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:22,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:22,931 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:22,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:22,936 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:23,815 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:24,818 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:25,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:25,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:54:25,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:54:25,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:54:28,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:28,969 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:28,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:28,974 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:29,821 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:30,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:30,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:34,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:34,294 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:34,294 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:34,299 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:34,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:34,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:35,759 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:54:36,760 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:39,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:39,568 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:39,569 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:39,574 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:39,831 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:40,833 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:40,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:54:40,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:54:40,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:54:42,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:43,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:43,177 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:43,178 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:43,183 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:43,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:44,839 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:47,215 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:47,215 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:47,216 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:47,221 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:47,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:48,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:48,843 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:51,611 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:51,612 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:51,612 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:51,617 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:51,845 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:52,847 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:53,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:55,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:55,238 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:55,238 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:55,243 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:55,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:54:55,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:54:55,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:54:55,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:54:56,851 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:54:59,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:59,720 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:59,721 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:54:59,721 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:59,726 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:59,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:00,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:04,727 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:05,761 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:55:09,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:10,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:10,286 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:10,286 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:10,291 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:10,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:10,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:10,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:55:10,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:55:10,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:55:15,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:15,045 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:15,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:15,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:15,050 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:15,868 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:16,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:20,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:21,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:21,319 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:21,320 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:21,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:21,874 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:22,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:25,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:25,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:55:25,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:55:25,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:55:26,116 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:26,240 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:26,240 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:26,245 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:26,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:26,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:30,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:30,270 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:30,271 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:30,276 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:30,885 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:30,885 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:31,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:34,340 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:34,341 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:34,341 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:34,346 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:34,889 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:34,889 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:35,763 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:55:36,764 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:38,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:38,016 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:38,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:38,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:38,893 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:38,893 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:40,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:55:40,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:55:40,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:55:42,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:43,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:43,606 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:43,606 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:43,611 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:43,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:44,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:47,611 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:48,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:48,969 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:48,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:48,975 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:49,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:50,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:52,975 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:53,732 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:53,733 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:53,733 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:53,738 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:53,907 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:55:54,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:55:55,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:55:55,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:55:55,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:55:58,180 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:58,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:58,950 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:55:58,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:58,956 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:59,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:00,916 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:03,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:03,580 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:03,580 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:03,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:03,585 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:03,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:04,920 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:05,765 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:56:07,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:07,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:07,438 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:07,442 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:07,922 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:08,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:09,443 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:10,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:56:10,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:56:10,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:56:11,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:11,230 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:11,231 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:11,236 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:11,927 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:12,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:14,327 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:14,328 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:14,328 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:14,334 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:14,932 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:14,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:15,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:18,505 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:18,506 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:18,506 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:18,511 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:18,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:20,512 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:20,939 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:24,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:24,364 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:24,364 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:24,369 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:24,943 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:24,943 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:25,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:56:25,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:56:25,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:56:26,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:28,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:28,824 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:28,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:28,829 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:28,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:30,948 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:31,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:32,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:32,797 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:32,798 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:32,803 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:32,949 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:34,953 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:35,767 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:56:37,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:37,095 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:37,095 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:37,095 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:37,100 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:37,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:38,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:40,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:56:40,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:56:40,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:56:41,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:41,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:41,853 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:41,858 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:41,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:42,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:42,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:45,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:45,730 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:45,730 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:45,735 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:45,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:46,966 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:48,735 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:49,564 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:49,565 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:49,565 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:49,570 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:49,968 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:50,971 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:53,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:53,324 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:53,324 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:53,329 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:53,972 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:54,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:54,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:55,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:56:55,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:56:55,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:56:58,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:58,336 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:56:58,336 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:58,341 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:58,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:56:58,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:56:59,342 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:02,721 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:02,722 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:02,722 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:02,727 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:02,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:04,727 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:04,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:05,769 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:57:09,771 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:10,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:10,244 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:10,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:10,249 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:10,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:57:10,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:57:10,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:57:10,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:10,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:13,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:13,964 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:13,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:13,969 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:13,993 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:14,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:14,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:17,839 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:17,839 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:17,840 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:17,844 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:17,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:19,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:20,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:22,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:22,509 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:22,509 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:22,514 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:23,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:25,006 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:25,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:25,943 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:25,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:25,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:25,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:57:25,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:57:25,948 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:25,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:57:26,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:27,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:30,145 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:30,146 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:30,146 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:30,151 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:31,014 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:31,014 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:31,152 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:35,775 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:57:35,847 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:35,848 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:35,848 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:35,854 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:36,016 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:36,854 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:37,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:39,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:39,827 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:39,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:39,831 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:40,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:40,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:57:40,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:57:40,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:57:41,023 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:42,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:43,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:43,910 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:43,910 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:43,915 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:44,026 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:45,029 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:47,916 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:48,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:48,440 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:48,440 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:48,446 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:49,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:49,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:52,122 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:52,123 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:52,123 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:52,128 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:53,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:53,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:57:53,128 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:55,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:57:55,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:57:55,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:57:56,897 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:56,898 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:57:56,898 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:56,903 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:57,040 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:57:58,903 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:59,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:01,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:01,274 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:01,275 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:01,279 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:02,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:03,049 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:04,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:05,774 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:58:07,739 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:07,740 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:07,740 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:07,745 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:08,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:09,055 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:09,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:10,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:58:10,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:58:10,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:58:11,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:11,384 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:11,384 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:11,389 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:12,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:13,059 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:15,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:17,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:17,231 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:17,231 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:17,236 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:18,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:19,065 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:21,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:21,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:21,238 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:21,238 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:21,243 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:22,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:23,069 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:24,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:24,825 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:24,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:24,830 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:25,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:25,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:58:25,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:58:25,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:58:27,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:27,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:28,979 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:28,979 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:28,980 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:28,984 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:29,074 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:31,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:32,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:33,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:33,942 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:33,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:33,947 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:34,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:35,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:35,779 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:58:37,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:37,576 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:37,577 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:37,582 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:38,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:38,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:39,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:40,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:58:40,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:58:40,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:58:43,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:43,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:43,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:43,473 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:44,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:44,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:45,092 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:47,196 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:47,197 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:47,197 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:47,202 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:48,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:49,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:50,203 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:50,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:50,815 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:50,815 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:50,820 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:51,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:53,101 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:58:55,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:55,214 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:58:55,214 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:55,214 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:55,219 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:55,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:58:55,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:58:55,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:58:56,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:58:57,105 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:00,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:00,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:00,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:00,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:00,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:01,110 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:01,111 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:05,445 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:05,778 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:59:06,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:06,130 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:06,130 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:06,134 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:07,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:07,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:09,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:09,996 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:09,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:10,001 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:10,118 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:10,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:59:10,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:59:10,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:59:11,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:11,182 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:16,182 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:16,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:16,704 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:16,704 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:16,709 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:17,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:19,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:21,710 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:23,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:23,094 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:23,094 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:23,099 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:23,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:25,139 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:25,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:59:25,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:59:25,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:59:27,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:27,548 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:27,549 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:27,549 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:27,554 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:28,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:29,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:31,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:31,967 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:31,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:31,972 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:32,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:32,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:33,150 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:35,780 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 12:59:36,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:36,240 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:36,240 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:36,245 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:37,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:37,156 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:38,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:40,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:40,154 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:40,154 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:40,159 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:40,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:40,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:59:40,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:59:40,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:59:41,161 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:44,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:44,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:44,836 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:44,836 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:44,841 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:45,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:47,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:49,842 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:50,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:50,057 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:50,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:50,062 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:50,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:51,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 12:59:55,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:55,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:55,320 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 12:59:55,321 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:55,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:55,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:59:55,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:59:55,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:59:56,180 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 12:59:57,184 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:00,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:00,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:00,301 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:00,301 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:00,306 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:01,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:01,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:04,120 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:04,121 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:04,121 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:04,126 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:04,193 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:05,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:05,782 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:00:05,783 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:08,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:08,021 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:08,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:08,026 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:08,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:09,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:10,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:00:10,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:00:10,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:00:11,273 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:13,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:13,277 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:13,277 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:13,282 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:14,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:15,210 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:16,318 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:17,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:17,031 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:17,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:17,036 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:17,212 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:19,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:20,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:20,576 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:20,576 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:20,581 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:21,219 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:21,219 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:21,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:25,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:25,293 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:25,293 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:25,298 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:25,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:00:25,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:00:25,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:00:26,223 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:27,228 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:27,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:28,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:28,828 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:28,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:28,833 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:29,230 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:31,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:32,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:35,784 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:00:36,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:36,362 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:36,362 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:36,367 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:37,241 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:37,242 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:38,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:40,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:00:40,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:00:40,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:00:41,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:41,829 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:41,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:41,834 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:42,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:43,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:43,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:45,731 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:45,732 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:45,732 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:45,737 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:46,252 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:47,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:49,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:50,395 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:50,396 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:50,396 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:50,401 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:51,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:51,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:53,651 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:53,658 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:53,658 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:53,663 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:54,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:00:55,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:00:55,663 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:55,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:00:55,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:00:55,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:00:59,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:59,273 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:00:59,273 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:59,278 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:00,272 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:01,278 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:01,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:03,555 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:03,556 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:03,556 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:03,561 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:04,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:05,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:05,786 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:01:06,787 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:07,708 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:07,709 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:07,709 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:07,714 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:08,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:09,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:10,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:01:10,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:01:10,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:01:12,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:13,743 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:13,743 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:13,744 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:13,748 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:14,292 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:15,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:17,749 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:19,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:19,836 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:19,836 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:19,841 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:20,301 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:21,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:22,842 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:24,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:24,195 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:24,195 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:24,200 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:24,307 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:25,311 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:25,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:01:25,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:01:25,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:01:27,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:27,938 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:27,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:27,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:27,943 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:28,313 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:29,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:31,890 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:31,890 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:31,891 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:31,896 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:32,318 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:33,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:33,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:35,454 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:35,454 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:35,455 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:35,460 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:35,788 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:01:36,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:37,328 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:39,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:40,410 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:40,411 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:40,411 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:40,416 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:40,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:01:40,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:01:40,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:01:41,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:41,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:43,707 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:43,708 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:43,708 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:43,713 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:44,337 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:45,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:45,714 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:49,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:49,561 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:49,561 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:49,566 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:50,344 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:51,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:51,567 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:53,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:53,321 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:53,322 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:53,327 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:53,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:55,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:01:55,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:01:55,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:01:55,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:01:57,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:57,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:57,409 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:01:57,410 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:57,415 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:58,356 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:01:59,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:01,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:01,242 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:01,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:01,247 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:01,362 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:02,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:03,366 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:05,372 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:05,373 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:05,373 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:05,378 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:05,794 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:02:06,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:07,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:07,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:09,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:09,832 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:09,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:09,837 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:10,374 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:10,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:02:10,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:02:10,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:02:11,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:13,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:14,204 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:14,205 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:14,205 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:14,210 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:14,380 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:15,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:18,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:18,171 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:18,172 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:18,177 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:18,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:19,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:19,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:23,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:23,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:23,259 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:23,264 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:23,393 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:24,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:25,398 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:25,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:02:25,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:02:25,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:02:28,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:28,577 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:28,577 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:28,582 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:29,403 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:29,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:29,583 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:32,070 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:32,071 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:32,071 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:32,077 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:32,405 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:33,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:35,077 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:35,793 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:02:36,684 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:36,685 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:36,685 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:36,690 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:37,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:37,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:40,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:40,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:02:40,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:02:40,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:02:45,123 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:45,124 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:45,124 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:45,130 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:45,424 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:46,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:47,429 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:50,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:50,739 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:50,739 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:50,744 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:51,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:51,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:51,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:55,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:55,040 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:55,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:55,046 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:55,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:02:55,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:02:55,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:02:55,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:02:57,220 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:57,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:02:59,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:59,203 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:02:59,203 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:59,208 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:59,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:03:01,450 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:03:03,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:03,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:03,860 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:03:03,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:03,866 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:04,452 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:03:05,454 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:03:05,798 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:03:08,799 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:08,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:08,830 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:03:08,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:08,835 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:09,458 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:03:09,458 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:03:10,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:03:10,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:03:10,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:03:14,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:18,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:18,262 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:03:18,262 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:18,267 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:18,465 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:03:19,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:19,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:03:23,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:23,454 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:03:23,454 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:23,459 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:23,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:03:24,459 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:25,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:03:25,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:03:25,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:03:25,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:03:27,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:27,442 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:03:27,443 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:27,448 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:27,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:03:29,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:03:30,448 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:31,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:31,798 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:03:31,798 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:31,803 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:32,479 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:03:33,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:03:35,797 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:03:35,798 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:38,455 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:38,455 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:03:38,456 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:38,461 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:38,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:03:39,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:03:40,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:03:40,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:03:40,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:03:41,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:44,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:44,716 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:03:44,717 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:44,722 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:45,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:03:45,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:03:46,722 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:51,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:53,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:53,178 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:03:53,178 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:53,183 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:53,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:03:55,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:03:55,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:03:55,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:03:55,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:03:57,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:02,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:02,234 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:02,235 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:02,240 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:02,507 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:03,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:03,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:04:05,799 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:04:05,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:05,974 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:05,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:05,979 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:06,511 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:07,514 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:04:08,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:10,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:04:10,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:04:10,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:04:11,505 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:11,506 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:11,506 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:11,511 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:11,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:13,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:04:14,511 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:16,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:16,047 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:16,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:16,053 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:16,522 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:17,524 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:04:20,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:20,743 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:20,744 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:20,744 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:20,750 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:21,529 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:21,529 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:04:25,751 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:25,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:04:25,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:04:25,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:04:27,626 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:27,627 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:27,627 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:27,632 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:28,533 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:29,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:04:31,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:33,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:33,742 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:33,742 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:33,747 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:34,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:35,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:04:35,801 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:04:36,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:38,135 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:38,136 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:38,136 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:38,141 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:38,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:39,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:04:40,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:04:40,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:04:40,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:04:42,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:45,123 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:45,124 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:45,124 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:45,129 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:45,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:47,554 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:04:48,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:49,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:49,575 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:49,575 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:49,580 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:50,555 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:51,558 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:04:53,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:55,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:04:55,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:04:55,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:04:57,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:57,881 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:04:57,881 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:57,886 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:58,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:04:58,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:59,564 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:02,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:02,641 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:02,642 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:02,647 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:03,570 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:03,570 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:04,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:05,803 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:05:08,642 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:08,643 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:08,643 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:08,648 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:09,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:09,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:09,649 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:10,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:05:10,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:05:10,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:05:12,863 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:12,864 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:12,864 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:12,870 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:13,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:13,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:14,870 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:17,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:17,459 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:17,459 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:17,464 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:17,583 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:19,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:20,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:25,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:25,915 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:25,916 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:25,916 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:25,921 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:25,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:05:25,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:05:25,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:05:26,591 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:27,593 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:29,411 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:29,411 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:29,412 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:29,417 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:29,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:31,417 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:31,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:33,454 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:33,455 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:33,455 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:33,460 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:33,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:35,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:35,809 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:05:36,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:38,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:38,320 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:38,320 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:38,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:38,604 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:39,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:40,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:05:40,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:05:40,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:05:42,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:46,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:46,563 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:46,563 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:46,568 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:46,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:47,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:47,615 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:52,459 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:52,460 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:52,460 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:52,465 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:52,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:53,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:53,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:55,867 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:55,868 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:05:55,868 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:55,873 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:55,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:05:55,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:05:55,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:05:56,622 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:05:57,624 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:05:59,215 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:04,216 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:04,388 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:04,389 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:06:04,389 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:04,395 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:04,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:06:05,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:06:05,807 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:06:09,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:10,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:06:10,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:06:10,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:06:12,707 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:12,708 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:06:12,708 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:12,714 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:13,638 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:06:13,638 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:06:15,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:17,125 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:17,126 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:06:17,126 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:17,131 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:17,641 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:06:19,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:06:21,132 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:23,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:23,888 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:06:23,888 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:23,893 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:24,647 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:06:25,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:06:25,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:06:25,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:06:25,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:06:26,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:27,498 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:27,499 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:06:27,500 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:27,505 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:27,651 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:06:29,654 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:06:31,506 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:32,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:32,224 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:06:32,225 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:32,230 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:32,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:06:33,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:06:35,812 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:06:36,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:36,434 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:06:36,434 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:36,439 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:36,661 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:06:37,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:37,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:06:40,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:06:40,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:06:40,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:06:43,192 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:43,796 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:43,797 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:06:43,797 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:43,802 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:44,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:06:45,672 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:06:47,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:47,516 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:06:47,517 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:47,521 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:47,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:06:48,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:49,674 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:06:51,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:51,681 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:06:51,682 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:51,687 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:52,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:06:53,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:06:53,687 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:55,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:06:55,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:06:55,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:06:59,173 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:00,351 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:00,351 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:00,352 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:00,357 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:00,683 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:01,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:04,358 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:04,752 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:04,753 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:04,753 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:04,759 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:05,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:05,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:05,812 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:07:09,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:09,390 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:09,390 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:09,390 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:09,395 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:09,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:10,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:07:10,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:07:10,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:07:11,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:13,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:13,327 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:13,327 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:13,333 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:13,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:15,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:15,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:17,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:17,284 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:17,284 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:17,289 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:17,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:19,708 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:21,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:22,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:22,130 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:22,130 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:22,135 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:22,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:23,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:25,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:25,883 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:25,883 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:25,888 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:25,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:07:25,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:07:25,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:07:26,714 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:27,175 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:27,716 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:31,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:31,301 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:31,301 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:31,306 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:31,719 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:32,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:33,722 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:34,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:34,925 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:34,925 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:34,930 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:35,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:35,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:35,814 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:07:37,815 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:38,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:38,969 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:38,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:38,974 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:39,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:39,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:40,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:07:40,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:07:40,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:07:42,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:42,292 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:42,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:42,297 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:42,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:43,298 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:43,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:46,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:46,704 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:46,704 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:46,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:46,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:47,739 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:48,710 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:50,092 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:50,093 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:50,093 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:50,098 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:50,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:51,744 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:54,099 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:54,351 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:54,352 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:54,352 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:54,357 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:54,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:07:55,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:07:55,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:07:55,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:07:55,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:07:59,189 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:59,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:59,502 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:07:59,502 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:59,507 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:59,751 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:01,755 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:04,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:04,290 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:04,290 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:04,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:04,295 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:04,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:05,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:05,816 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:08:08,716 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:08,717 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:08,717 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:08,722 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:08,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:09,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:09,765 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:10,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:08:10,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:08:10,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:08:13,712 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:13,713 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:13,713 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:13,719 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:13,767 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:15,719 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:15,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:19,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:19,989 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:19,989 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:19,994 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:20,774 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:20,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:21,777 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:23,498 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:23,499 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:23,500 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:23,505 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:23,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:25,781 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:25,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:08:25,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:08:25,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:08:26,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:28,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:28,254 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:28,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:28,259 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:28,783 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:29,786 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:31,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:33,108 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:33,109 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:33,109 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:33,114 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:33,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:33,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:35,818 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:08:36,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:37,191 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:37,192 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:37,192 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:37,197 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:37,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:37,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:40,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:40,903 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:40,903 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:40,908 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:40,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:08:40,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:08:40,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:08:41,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:41,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:42,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:44,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:44,959 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:44,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:44,964 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:45,804 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:45,804 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:47,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:49,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:49,263 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:49,263 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:49,268 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:49,808 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:49,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:53,138 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:53,139 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:53,139 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:53,139 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:53,144 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:53,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:53,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:08:55,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:08:55,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:08:55,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:08:58,203 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:58,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:58,559 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:08:58,560 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:58,565 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:58,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:08:59,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:02,719 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:02,720 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:02,720 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:02,725 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:02,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:03,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:03,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:05,820 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:09:08,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:10,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:09:10,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:09:10,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:09:11,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:11,207 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:11,207 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:11,213 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:11,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:11,833 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:14,214 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:15,111 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:15,112 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:15,112 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:15,117 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:15,837 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:15,837 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:19,152 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:19,153 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:19,153 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:19,158 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:19,841 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:19,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:20,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:25,159 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:25,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:09:25,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:09:25,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:09:27,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:27,615 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:27,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:27,620 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:27,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:29,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:30,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:31,875 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:31,876 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:31,876 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:31,881 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:32,851 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:33,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:35,822 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:09:35,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:37,528 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:37,529 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:37,529 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:37,534 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:37,856 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:39,862 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:40,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:09:40,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:09:40,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:09:41,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:43,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:43,386 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:43,386 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:43,391 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:43,866 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:45,869 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:46,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:47,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:47,130 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:47,130 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:47,135 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:47,872 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:47,872 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:51,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:51,137 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:51,137 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:51,142 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:51,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:51,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:52,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:55,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:55,137 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:09:55,137 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:55,142 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:55,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:09:55,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:09:55,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:09:55,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:09:55,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:09:57,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:00,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:00,944 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:00,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:00,949 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:01,888 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:01,888 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:02,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:05,830 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:10:06,505 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:06,505 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:06,506 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:06,511 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:06,891 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:07,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:08,511 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:10,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:10:10,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:10:10,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:10:12,352 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:12,352 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:12,353 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:12,358 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:12,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:13,900 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:14,358 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:16,113 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:16,114 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:16,114 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:16,119 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:16,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:17,904 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:20,120 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:24,571 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:24,572 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:24,572 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:24,577 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:24,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:25,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:25,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:25,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:10:25,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:10:25,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:10:29,799 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:29,799 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:29,800 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:29,805 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:29,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:30,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:31,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:33,525 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:33,526 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:33,526 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:33,531 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:33,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:35,827 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:10:35,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:35,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:37,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:37,097 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:37,097 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:37,102 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:37,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:37,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:40,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:10:40,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:10:40,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:10:41,179 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:43,175 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:43,176 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:43,176 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:43,181 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:43,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:43,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:46,181 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:46,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:46,825 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:46,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:46,830 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:46,932 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:47,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:50,228 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:50,229 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:50,229 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:50,234 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:50,936 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:51,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:51,939 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:55,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:55,751 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:55,751 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:55,755 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:55,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:10:55,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:10:55,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:10:55,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:10:57,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:57,944 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:10:59,870 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:59,870 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:10:59,871 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:59,876 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:59,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:01,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:02,876 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:04,413 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:04,414 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:04,414 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:04,419 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:04,953 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:05,832 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:11:05,956 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:08,118 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:08,118 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:08,118 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:08,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:08,123 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:08,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:09,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:10,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:11:10,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:11:10,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:11:12,899 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:12,899 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:12,900 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:12,904 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:12,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:13,905 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:13,965 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:17,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:17,047 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:17,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:17,052 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:17,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:17,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:19,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:21,775 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:21,775 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:21,776 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:21,781 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:21,971 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:23,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:24,781 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:25,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:11:25,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:11:25,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:11:26,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:26,191 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:26,191 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:26,196 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:26,976 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:27,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:29,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:29,461 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:29,462 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:29,466 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:29,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:29,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:30,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:35,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:35,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:35,542 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:35,542 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:35,547 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:35,831 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:11:35,985 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:37,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:39,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:39,322 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:39,322 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:39,327 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:39,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:39,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:40,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:11:40,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:11:40,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:11:41,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:44,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:44,306 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:44,307 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:44,311 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:44,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:45,999 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:46,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:48,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:48,471 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:48,471 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:48,476 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:49,000 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:50,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:51,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:51,987 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:51,988 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:51,988 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:51,993 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:52,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:54,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:11:55,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:11:55,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:11:55,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:11:56,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:56,393 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:11:56,393 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:56,398 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:57,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:11:57,398 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:58,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:00,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:00,097 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:00,097 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:00,102 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:01,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:02,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:03,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:05,833 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:12:06,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:06,231 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:06,231 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:06,236 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:07,020 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:08,023 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:08,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:10,660 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:10,660 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:10,660 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:10,665 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:10,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:12:10,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:12:10,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:12:11,025 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:12,026 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:13,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:14,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:14,465 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:14,465 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:14,471 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:15,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:16,030 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:18,472 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:18,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:18,887 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:18,887 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:18,892 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:19,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:20,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:23,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:25,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:25,483 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:25,483 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:25,488 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:25,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:12:25,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:12:25,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:12:26,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:26,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:29,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:29,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:29,862 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:29,862 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:29,867 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:30,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:32,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:34,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:34,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:34,964 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:34,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:34,969 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:35,048 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:35,835 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:12:36,051 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:39,398 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:39,399 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:39,399 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:39,404 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:40,055 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:40,055 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:40,405 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:40,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:12:40,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:12:40,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:12:43,122 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:43,123 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:43,123 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:43,128 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:44,060 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:44,060 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:46,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:46,708 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:46,709 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:46,709 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:46,717 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:47,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:48,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:50,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:50,747 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:50,747 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:50,752 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:51,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:51,752 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:52,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:54,904 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:54,905 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:12:54,905 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:54,909 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:55,068 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:12:55,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:12:55,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:12:55,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:12:56,071 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:12:57,216 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:02,217 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:03,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:03,434 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:03,434 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:03,438 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:04,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:04,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:05,837 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:13:07,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:09,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:09,224 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:09,224 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:09,230 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:10,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:10,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:10,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:13:10,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:13:10,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:13:12,628 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:12,629 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:12,629 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:12,634 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:13,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:13,634 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:14,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:16,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:16,674 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:16,674 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:16,679 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:17,091 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:18,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:18,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:20,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:20,218 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:20,219 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:20,223 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:21,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:22,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:24,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:25,288 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:25,288 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:25,289 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:25,293 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:25,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:13:25,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:13:25,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:13:26,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:26,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:29,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:29,041 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:29,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:29,046 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:29,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:30,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:30,106 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:34,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:34,442 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:34,442 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:34,446 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:35,109 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:35,447 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:35,839 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:13:36,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:39,784 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:39,785 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:39,785 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:39,790 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:40,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:40,791 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:40,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:13:40,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:13:40,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:13:42,117 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:43,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:43,338 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:43,338 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:43,343 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:44,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:44,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:46,344 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:47,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:47,292 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:47,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:47,297 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:48,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:48,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:51,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:51,505 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:51,505 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:51,505 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:51,510 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:52,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:52,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:13:55,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:13:55,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:13:55,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:13:57,150 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:57,151 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:13:57,151 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:57,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:57,156 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:58,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:13:58,136 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:01,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:01,962 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:01,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:01,967 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:02,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:02,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:04,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:05,841 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:14:06,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:06,510 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:06,510 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:06,515 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:07,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:08,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:08,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:10,557 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:10,558 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:10,558 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:10,563 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:10,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:14:10,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:14:10,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:14:11,150 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:12,153 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:14,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:14,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:14,790 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:14,790 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:14,795 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:15,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:16,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:18,850 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:18,851 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:18,851 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:18,856 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:19,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:19,856 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:20,162 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:24,188 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:24,188 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:24,189 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:24,194 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:25,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:25,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:25,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:14:25,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:14:25,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:14:26,167 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:29,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:29,062 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:29,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:29,067 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:29,169 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:30,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:31,068 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:33,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:33,052 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:33,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:33,058 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:33,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:34,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:35,847 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:14:36,848 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:38,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:38,751 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:38,751 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:38,756 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:39,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:40,181 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:40,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:14:40,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:14:40,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:14:42,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:42,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:42,648 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:42,648 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:42,653 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:43,183 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:44,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:47,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:47,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:47,745 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:47,745 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:47,750 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:48,189 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:50,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:51,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:51,822 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:51,822 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:51,827 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:52,194 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:52,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:54,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:55,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:14:55,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:14:55,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:14:56,193 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:56,200 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:14:56,200 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:56,205 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:57,199 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:14:58,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:14:58,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:02,216 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:02,216 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:02,216 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:02,221 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:03,205 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:03,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:04,208 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:05,846 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:15:05,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:05,850 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:05,850 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:05,855 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:06,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:08,212 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:08,855 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:10,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:10,367 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:10,367 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:10,372 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:10,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:15:10,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:15:10,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:15:11,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:12,217 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:14,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:16,288 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:16,288 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:16,289 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:16,293 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:17,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:18,223 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:19,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:21,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:21,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:21,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:21,996 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:22,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:24,228 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:24,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:25,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:15:25,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:15:25,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:15:26,800 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:26,800 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:26,801 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:26,806 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:27,230 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:28,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:30,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:32,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:32,470 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:32,470 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:32,475 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:33,236 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:34,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:35,851 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:15:35,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:36,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:36,617 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:36,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:36,622 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:37,240 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:38,243 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:40,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:15:40,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:15:40,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:15:41,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:41,442 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:41,443 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:41,443 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:41,448 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:42,245 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:42,245 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:45,204 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:45,205 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:45,205 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:45,210 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:45,248 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:46,210 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:46,250 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:51,211 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:51,668 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:51,669 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:51,669 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:51,674 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:52,255 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:15:52,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:15:55,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:15:55,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:15:55,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:15:56,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:58,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:58,103 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:15:58,103 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:58,108 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:58,259 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:00,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:01,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:01,910 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:01,911 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:01,911 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:01,916 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:02,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:04,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:05,850 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:16:06,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:06,074 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:06,074 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:06,079 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:06,269 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:07,079 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:08,272 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:10,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:10,583 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:10,583 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:10,588 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:10,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:16:10,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:16:10,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:16:11,274 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:12,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:12,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:16,400 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:16,401 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:16,401 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:16,406 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:17,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:17,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:18,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:20,120 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:20,120 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:20,121 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:20,126 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:20,283 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:22,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:23,127 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:24,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:24,727 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:24,727 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:24,732 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:25,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:25,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:16:25,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:16:25,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:16:26,291 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:28,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:30,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:30,683 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:30,684 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:30,688 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:31,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:32,297 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:33,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:35,134 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:35,135 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:35,135 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:35,140 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:35,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:35,856 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:16:36,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:38,857 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:40,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:40,332 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:40,333 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:40,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:40,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:16:40,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:16:40,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:16:41,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:42,307 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:44,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:46,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:46,948 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:46,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:46,954 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:47,311 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:48,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:49,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:51,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:51,548 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:51,548 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:51,553 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:52,318 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:52,318 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:55,342 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:55,343 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:55,343 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:55,343 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:55,348 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:55,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:16:55,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:16:55,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:16:56,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:16:56,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:16:59,711 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:59,711 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:16:59,712 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:59,717 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:00,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:00,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:00,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:04,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:04,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:04,697 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:04,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:05,331 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:05,855 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:17:05,856 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:06,333 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:08,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:08,374 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:08,374 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:08,379 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:09,335 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:10,338 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:10,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:17:10,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:17:10,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:17:11,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:12,416 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:12,417 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:12,417 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:12,422 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:13,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:14,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:16,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:16,054 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:16,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:16,059 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:16,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:17,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:18,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:21,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:21,012 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:21,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:21,017 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:21,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:22,351 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:23,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:25,169 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:25,169 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:25,170 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:25,175 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:25,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:25,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:17:25,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:17:25,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:17:26,355 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:28,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:29,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:29,292 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:29,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:29,297 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:29,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:30,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:33,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:33,023 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:33,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:33,029 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:33,361 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:34,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:34,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:35,856 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:17:37,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:37,386 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:37,386 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:37,391 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:38,368 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:38,368 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:39,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:40,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:17:40,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:17:40,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:17:42,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:42,877 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:42,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:42,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:43,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:44,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:44,884 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:46,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:46,415 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:46,415 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:46,420 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:47,376 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:48,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:50,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:54,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:54,581 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:54,582 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:54,587 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:55,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:17:55,587 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:55,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:17:55,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:17:55,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:17:56,385 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:17:58,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:58,266 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:17:58,266 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:58,271 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:58,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:00,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:01,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:02,478 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:02,479 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:02,479 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:02,484 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:03,391 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:04,393 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:05,858 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:18:06,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:06,251 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:06,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:06,257 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:06,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:07,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:08,398 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:09,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:09,954 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:09,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:09,959 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:10,399 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:10,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:18:10,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:18:10,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:18:12,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:13,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:15,370 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:15,371 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:15,371 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:15,376 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:15,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:16,407 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:18,376 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:19,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:19,921 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:19,922 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:19,927 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:20,411 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:20,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:23,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:23,895 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:23,896 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:23,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:23,901 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:24,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:24,416 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:25,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:18:25,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:18:25,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:18:27,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:27,556 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:27,556 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:27,561 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:28,420 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:28,420 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:29,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:31,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:31,911 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:31,911 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:31,916 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:32,424 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:32,424 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:34,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:35,860 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:18:36,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:36,286 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:36,286 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:36,291 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:36,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:38,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:40,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:40,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:18:40,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:18:40,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:18:44,786 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:44,787 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:44,787 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:44,792 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:45,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:45,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:46,437 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:49,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:49,550 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:49,550 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:49,555 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:50,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:50,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:51,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:54,643 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:54,644 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:54,645 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:54,649 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:55,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:18:55,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:18:55,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:18:55,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:18:56,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:18:57,191 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:58,687 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:58,687 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:18:58,688 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:58,693 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:59,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:00,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:02,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:03,739 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:03,740 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:03,740 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:03,745 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:04,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:04,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:05,868 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:19:07,870 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:10,755 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:10,756 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:10,756 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:10,761 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:10,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:19:10,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:19:10,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:19:11,461 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:12,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:13,191 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:15,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:15,349 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:15,349 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:15,354 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:15,465 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:16,468 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:18,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:19,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:19,336 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:19,336 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:19,341 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:19,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:20,472 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:23,569 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:23,570 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:23,570 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:23,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:23,575 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:24,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:24,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:25,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:19:25,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:19:25,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:19:29,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:29,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:29,681 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:29,682 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:29,687 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:30,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:30,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:34,687 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:35,865 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:19:38,147 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:38,148 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:38,148 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:38,153 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:38,487 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:40,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:40,490 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:40,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:19:40,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:19:40,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:19:41,324 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:41,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:41,325 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:41,330 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:41,490 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:42,493 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:45,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:46,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:46,829 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:46,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:46,834 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:47,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:48,500 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:50,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:50,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:50,981 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:50,981 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:50,986 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:51,501 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:19:52,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:19:55,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:19:55,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:19:55,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:19:56,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:59,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:59,546 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:19:59,546 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:59,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:00,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:00,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:01,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:03,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:03,563 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:03,563 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:03,568 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:04,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:04,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:05,870 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:20:06,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:09,103 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:09,104 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:09,104 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:09,110 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:09,518 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:10,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:10,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:20:10,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:20:10,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:20:12,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:13,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:13,035 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:13,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:13,040 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:13,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:14,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:18,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:18,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:18,770 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:18,771 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:18,776 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:19,529 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:20,532 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:23,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:24,147 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:24,148 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:24,148 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:24,153 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:24,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:25,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:20:25,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:20:25,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:20:26,537 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:28,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:28,018 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:28,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:28,023 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:28,540 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:28,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:29,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:33,676 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:33,677 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:33,677 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:33,682 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:34,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:34,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:34,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:35,869 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:20:37,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:37,814 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:37,814 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:37,819 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:38,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:38,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:39,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:40,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:20:40,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:20:40,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:20:42,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:42,255 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:42,256 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:42,261 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:42,552 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:44,556 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:45,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:46,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:46,836 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:46,836 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:46,841 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:47,558 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:48,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:50,593 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:50,594 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:50,594 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:50,594 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:50,600 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:51,563 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:20:52,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:20:55,601 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:55,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:20:55,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:20:55,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:20:58,211 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:58,212 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:20:58,212 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:58,217 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:58,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:00,572 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:01,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:02,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:02,620 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:02,621 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:02,626 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:03,575 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:04,577 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:05,871 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:21:06,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:06,005 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:06,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:06,010 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:06,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:06,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:07,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:10,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:21:10,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:21:10,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:21:12,173 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:13,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:13,018 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:13,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:13,023 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:13,585 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:14,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:16,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:16,726 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:16,727 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:16,732 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:17,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:17,732 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:18,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:22,109 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:22,109 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:22,110 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:22,115 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:22,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:22,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:23,115 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:25,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:21:25,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:21:25,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:21:26,506 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:26,506 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:26,507 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:26,511 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:26,598 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:28,512 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:28,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:31,349 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:31,349 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:31,350 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:31,354 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:31,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:32,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:34,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:35,873 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:21:39,736 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:39,737 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:39,737 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:39,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:39,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:40,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:40,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:40,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:21:40,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:21:40,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:21:43,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:43,252 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:43,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:43,258 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:43,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:44,618 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:45,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:48,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:48,242 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:48,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:48,248 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:48,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:50,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:51,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:51,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:51,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:21:51,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:51,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:52,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:21:52,627 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:21:55,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:21:55,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:21:55,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:21:57,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:00,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:00,859 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:00,859 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:00,864 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:01,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:02,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:02,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:05,875 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:22:06,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:06,115 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:06,115 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:06,120 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:06,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:06,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:08,120 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:10,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:22:10,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:22:10,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:22:11,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:11,981 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:11,981 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:11,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:12,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:12,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:13,987 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:18,370 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:18,371 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:18,371 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:18,377 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:18,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:19,377 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:20,653 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:22,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:22,095 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:22,095 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:22,100 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:22,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:22,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:25,101 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:25,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:22:25,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:22:25,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:22:26,413 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:26,414 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:26,414 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:26,419 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:26,658 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:28,661 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:30,420 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:30,812 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:30,813 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:30,813 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:30,818 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:31,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:32,666 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:35,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:35,227 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:35,227 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:35,232 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:35,668 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:35,877 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:22:35,879 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:36,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:39,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:39,882 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:39,882 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:39,887 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:40,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:40,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:40,888 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:40,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:22:40,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:22:40,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:22:43,454 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:43,455 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:43,456 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:43,461 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:43,677 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:44,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:46,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:47,898 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:47,899 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:47,900 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:47,904 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:48,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:48,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:51,905 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:52,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:52,305 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:52,305 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:52,311 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:52,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:54,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:55,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:22:55,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:22:55,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:22:56,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:56,210 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:56,210 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:56,215 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:56,693 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:22:56,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:22:57,216 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:59,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:59,873 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:22:59,874 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:59,878 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:00,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:00,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:23:02,879 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:04,611 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:04,611 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:23:04,612 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:04,616 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:04,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:05,880 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:23:06,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:23:07,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:10,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:10,682 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:23:10,682 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:10,687 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:10,706 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:10,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:23:10,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:23:10,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:23:12,710 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:23:13,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:18,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:18,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:18,948 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:23:18,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:18,953 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:19,714 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:20,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:23:23,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:25,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:23:25,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:23:25,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:23:27,719 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:27,720 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:23:27,721 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:27,725 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:28,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:28,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:23:29,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:31,774 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:31,775 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:23:31,775 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:31,780 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:32,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:32,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:23:34,781 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:35,514 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:35,515 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:23:35,515 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:35,520 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:35,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:35,885 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:23:36,733 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:23:39,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:40,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:40,574 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:23:40,575 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:40,580 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:40,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:40,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:23:40,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:23:40,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:23:42,739 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:23:45,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:47,090 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:47,091 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:23:47,091 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:47,097 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:47,743 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:48,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:23:50,847 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:50,848 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:23:50,848 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:50,848 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:50,853 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:51,748 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:52,750 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:23:55,404 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:55,405 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:23:55,405 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:55,410 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:55,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:23:55,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:23:55,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:23:55,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:23:56,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:56,755 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:00,196 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:00,197 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:00,198 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:00,203 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:00,759 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:00,759 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:01,203 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:05,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:05,246 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:05,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:05,251 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:05,763 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:05,884 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:24:06,766 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:06,885 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:10,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:24:10,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:24:10,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:24:12,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:13,573 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:13,574 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:13,574 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:13,579 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:13,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:14,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:17,528 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:17,529 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:17,529 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:17,530 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:17,535 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:17,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:18,777 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:21,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:21,269 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:21,269 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:21,274 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:21,779 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:22,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:23,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:25,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:24:25,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:24:25,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:24:26,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:26,705 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:26,705 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:26,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:26,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:28,711 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:28,787 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:30,646 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:30,647 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:30,647 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:30,652 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:30,789 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:32,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:34,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:35,886 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:24:39,079 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:39,083 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:39,083 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:39,088 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:39,796 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:40,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:40,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:40,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:24:40,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:24:40,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:24:43,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:43,445 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:43,446 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:43,450 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:43,800 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:44,804 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:45,451 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:47,401 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:47,402 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:47,402 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:47,407 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:47,806 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:48,808 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:51,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:55,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:55,138 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:55,138 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:55,143 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:55,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:24:55,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:24:55,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:24:55,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:24:56,815 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:24:57,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:59,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:59,069 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:24:59,070 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:59,075 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:59,817 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:00,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:03,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:03,126 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:03,126 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:03,127 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:03,131 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:03,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:04,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:05,888 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:25:07,636 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:07,637 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:07,637 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:07,642 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:07,827 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:08,643 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:08,829 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:10,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:25:10,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:25:10,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:25:12,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:12,027 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:12,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:12,033 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:12,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:12,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:14,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:16,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:16,445 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:16,445 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:16,450 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:16,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:18,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:19,451 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:20,707 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:20,708 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:20,708 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:20,713 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:20,841 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:22,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:24,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:24,677 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:24,678 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:24,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:24,683 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:24,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:25,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:25:25,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:25:25,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:25:26,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:29,222 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:29,223 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:29,223 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:29,227 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:29,851 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:30,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:30,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:33,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:33,060 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:33,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:33,065 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:33,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:34,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:35,890 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:25:35,892 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:37,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:37,497 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:37,497 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:37,502 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:37,859 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:38,862 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:40,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:25:40,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:25:40,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:25:41,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:42,240 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:42,240 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:42,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:42,246 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:42,866 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:42,866 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:46,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:46,923 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:46,924 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:46,924 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:46,929 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:47,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:48,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:51,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:52,189 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:52,189 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:52,190 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:52,195 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:52,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:52,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:55,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:25:55,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:25:55,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:25:56,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:56,252 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:25:56,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:56,257 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:56,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:25:56,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:25:57,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:01,875 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:01,876 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:01,876 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:01,882 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:01,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:02,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:02,888 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:26:05,892 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:26:07,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:08,370 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:08,370 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:08,371 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:08,376 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:08,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:08,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:26:10,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:26:10,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:26:10,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:26:12,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:12,340 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:12,340 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:12,345 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:12,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:12,904 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:26:13,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:17,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:17,016 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:17,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:17,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:17,907 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:18,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:26:19,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:22,217 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:22,218 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:22,218 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:22,223 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:22,916 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:22,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:26:24,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:25,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:26:25,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:26:25,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:26:27,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:27,484 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:27,485 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:27,490 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:27,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:28,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:26:29,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:32,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:32,670 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:32,671 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:32,675 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:32,926 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:34,676 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:34,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:26:35,894 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:26:37,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:37,239 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:37,239 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:37,244 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:37,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:38,934 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:26:40,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:40,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:26:40,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:26:40,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:26:41,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:41,845 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:41,846 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:41,850 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:41,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:42,938 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:26:45,851 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:50,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:50,154 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:50,155 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:50,160 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:50,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:50,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:26:51,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:54,836 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:54,837 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:26:54,837 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:54,842 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:54,948 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:26:55,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:26:55,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:26:55,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:26:56,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:56,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:01,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:01,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:01,386 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:01,386 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:01,391 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:01,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:02,956 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:05,900 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:27:05,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:05,947 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:05,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:05,952 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:05,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:06,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:06,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:09,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:09,739 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:09,739 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:09,744 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:09,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:10,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:27:10,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:27:10,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:27:10,965 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:12,178 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:13,792 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:13,793 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:13,793 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:13,798 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:13,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:14,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:17,799 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:21,071 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:21,072 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:21,072 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:21,077 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:21,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:22,978 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:23,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:25,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:25,219 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:25,219 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:25,224 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:25,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:27:25,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:27:25,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:27:25,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:26,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:28,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:29,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:29,682 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:29,683 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:29,688 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:29,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:30,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:33,688 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:34,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:34,364 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:34,364 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:34,369 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:34,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:35,898 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:27:36,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:38,900 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:39,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:39,037 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:39,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:39,042 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:39,993 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:40,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:27:40,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:27:40,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:27:40,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:43,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:43,032 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:43,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:43,037 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:43,997 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:44,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:45,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:49,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:49,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:49,039 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:49,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:49,044 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:50,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:51,006 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:53,298 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:53,299 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:53,299 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:53,304 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:54,008 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:54,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:55,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:27:55,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:27:55,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:27:55,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:27:57,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:57,062 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:27:57,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:57,067 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:58,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:27:59,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:00,105 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:01,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:01,585 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:01,585 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:01,590 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:02,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:03,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:05,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:05,906 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:28:10,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:10,224 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:10,224 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:10,230 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:10,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:28:10,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:28:10,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:28:11,026 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:11,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:13,029 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:14,345 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:14,346 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:14,346 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:14,350 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:15,030 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:16,351 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:17,034 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:17,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:17,670 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:17,670 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:17,675 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:18,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:19,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:21,676 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:22,514 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:22,515 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:22,515 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:22,520 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:23,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:25,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:25,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:28:25,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:28:25,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:28:26,410 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:26,411 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:26,411 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:26,416 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:27,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:27,416 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:29,047 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:30,352 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:30,353 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:30,353 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:30,358 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:31,049 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:33,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:33,359 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:34,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:34,245 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:34,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:34,250 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:35,053 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:35,903 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:28:37,058 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:38,904 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:40,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:28:40,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:28:40,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:28:42,792 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:42,792 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:42,793 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:42,797 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:43,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:44,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:45,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:46,207 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:46,207 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:46,208 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:46,212 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:47,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:49,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:50,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:50,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:50,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:50,990 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:50,995 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:51,071 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:53,074 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:55,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:55,095 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:55,095 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:55,100 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:55,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:28:55,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:28:55,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:28:56,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:28:56,174 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:57,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:28:58,776 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:58,777 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:28:58,777 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:58,783 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:59,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:01,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:01,784 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:03,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:03,016 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:03,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:03,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:03,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:05,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:05,908 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:29:06,841 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:06,842 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:06,842 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:06,843 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:06,847 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:07,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:09,093 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:10,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:29:10,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:29:10,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:29:11,651 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:11,652 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:11,652 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:11,658 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:12,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:12,658 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:13,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:15,188 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:15,189 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:15,189 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:15,195 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:16,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:17,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:18,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:19,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:19,815 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:19,815 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:19,820 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:20,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:21,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:23,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:25,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:29:25,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:29:25,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:29:26,836 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:26,836 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:26,837 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:26,841 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:27,110 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:28,842 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:29,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:30,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:30,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:30,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:30,997 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:31,115 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:33,118 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:33,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:35,907 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:29:39,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:40,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:40,523 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:40,523 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:40,528 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:40,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:29:40,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:29:40,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:29:41,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:43,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:44,077 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:44,078 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:44,078 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:44,083 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:44,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:45,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:45,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:47,771 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:47,772 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:47,772 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:47,777 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:48,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:49,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:50,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:51,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:51,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:51,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:51,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:52,136 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:53,139 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:55,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:29:55,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:29:55,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:29:56,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:56,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:56,234 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:56,234 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:56,239 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:57,143 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:29:57,143 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:29:59,803 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:59,804 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:29:59,804 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:59,809 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:00,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:01,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:01,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:05,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:05,004 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:05,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:05,009 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:05,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:05,909 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:30:06,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:07,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:09,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:09,271 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:09,272 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:09,277 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:10,156 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:10,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:30:10,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:30:10,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:30:11,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:12,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:17,168 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:17,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:17,600 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:17,601 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:17,605 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:18,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:19,166 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:21,502 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:21,502 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:21,503 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:21,508 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:22,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:22,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:23,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:25,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:30:25,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:30:25,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:30:27,415 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:27,416 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:27,416 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:27,421 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:28,173 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:28,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:29,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:31,347 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:31,348 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:31,348 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:31,354 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:32,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:33,180 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:34,354 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:35,911 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:30:36,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:36,821 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:36,821 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:36,826 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:37,182 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:39,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:39,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:40,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:40,567 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:40,567 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:40,572 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:40,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:30:40,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:30:40,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:30:41,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:43,190 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:45,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:47,679 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:47,680 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:47,680 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:47,685 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:48,194 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:49,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:50,686 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:51,565 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:51,565 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:51,566 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:51,571 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:52,199 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:53,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:30:55,426 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:55,427 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:30:55,427 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:55,432 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:55,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:30:55,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:30:55,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:30:56,203 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:30:56,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:57,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:01,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:03,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:03,859 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:03,859 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:03,867 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:04,211 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:31:05,213 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:05,913 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:31:06,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:09,663 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:09,664 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:09,664 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:09,669 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:10,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:31:10,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:31:10,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:31:10,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:31:11,219 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:12,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:13,550 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:13,551 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:13,552 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:13,557 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:14,221 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:31:15,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:17,557 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:19,586 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:19,586 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:19,587 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:19,592 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:20,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:31:21,230 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:22,593 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:25,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:31:25,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:31:25,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:31:26,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:26,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:26,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:26,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:27,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:31:27,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:29,237 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:32,422 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:32,423 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:32,423 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:32,428 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:33,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:31:33,429 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:35,242 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:35,918 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:31:36,651 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:36,652 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:36,652 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:36,657 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:37,243 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:31:38,658 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:39,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:40,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:31:40,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:31:40,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:31:44,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:45,771 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:45,772 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:45,772 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:45,777 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:46,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:31:47,254 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:49,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:49,702 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:49,703 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:49,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:49,708 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:50,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:31:51,258 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:54,709 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:54,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:54,846 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:54,846 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:54,851 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:55,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:31:55,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:31:55,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:31:55,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:31:57,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:31:58,606 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:58,607 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:31:58,607 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:58,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:59,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:00,613 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:01,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:03,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:03,226 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:03,226 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:03,231 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:03,271 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:05,274 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:05,917 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:32:05,918 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:07,978 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:07,979 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:07,979 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:07,984 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:08,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:09,278 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:10,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:32:10,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:32:10,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:32:11,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:16,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:16,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:16,464 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:16,464 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:16,469 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:17,283 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:19,287 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:21,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:24,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:24,972 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:24,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:24,977 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:25,291 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:25,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:32:25,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:32:25,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:32:27,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:27,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:32,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:33,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:33,282 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:33,282 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:33,288 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:33,297 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:35,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:35,922 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:32:37,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:37,367 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:37,367 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:37,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:37,372 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:38,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:39,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:40,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:32:40,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:32:40,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:32:41,127 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:41,212 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:41,213 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:41,218 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:41,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:43,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:43,309 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:47,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:47,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:47,296 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:47,301 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:47,312 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:48,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:49,315 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:50,675 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:50,676 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:50,677 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:50,681 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:51,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:53,320 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:53,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:55,216 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:55,217 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:55,217 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:55,222 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:55,321 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:32:55,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:32:55,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:32:55,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:32:57,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:32:58,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:58,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:32:58,809 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:58,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:58,814 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:59,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:01,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:03,072 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:03,073 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:03,073 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:03,078 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:03,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:04,079 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:05,332 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:05,921 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:33:06,624 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:06,625 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:06,625 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:06,630 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:07,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:09,337 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:09,631 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:10,784 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:10,785 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:10,785 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:10,790 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:10,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:33:10,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:33:10,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:33:11,338 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:13,341 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:15,170 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:15,171 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:15,171 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:15,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:15,176 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:15,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:17,346 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:20,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:20,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:20,291 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:20,291 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:20,296 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:20,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:21,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:24,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:24,120 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:24,120 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:24,125 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:24,352 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:25,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:25,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:33:25,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:33:25,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:33:26,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:29,817 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:29,817 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:29,818 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:29,823 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:30,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:31,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:31,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:33,975 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:33,976 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:33,976 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:33,981 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:34,361 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:35,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:35,924 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:33:36,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:38,128 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:38,128 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:38,129 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:38,134 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:38,366 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:39,368 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:40,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:33:40,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:33:40,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:33:42,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:42,136 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:42,137 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:42,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:42,142 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:42,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:43,372 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:45,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:45,956 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:45,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:45,961 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:46,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:47,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:47,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:50,232 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:50,233 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:50,233 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:50,238 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:50,380 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:51,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:33:53,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:55,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:33:55,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:33:55,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:33:58,530 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:58,531 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:33:58,532 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:58,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:58,537 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:59,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:33:59,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:02,472 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:02,473 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:02,473 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:02,479 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:03,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:03,395 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:04,480 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:05,926 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:34:07,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:07,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:07,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:07,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:08,397 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:09,400 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:09,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:10,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:34:10,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:34:10,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:34:13,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:13,497 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:13,497 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:13,502 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:14,403 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:15,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:15,537 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:20,537 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:20,862 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:20,863 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:20,863 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:20,868 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:21,407 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:23,410 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:25,869 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:25,900 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:25,901 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:25,901 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:25,906 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:25,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:34:25,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:34:25,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:34:26,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:27,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:29,572 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:29,573 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:29,573 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:29,578 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:30,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:31,419 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:31,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:33,459 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:33,460 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:33,460 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:33,465 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:34,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:35,423 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:35,928 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:34:36,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:40,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:34:40,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:34:40,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:34:41,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:41,954 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:41,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:41,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:41,959 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:42,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:43,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:45,891 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:45,892 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:45,892 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:45,897 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:46,433 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:47,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:47,898 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:52,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:52,057 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:52,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:52,063 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:52,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:53,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:53,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:55,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:34:55,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:34:55,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:34:56,571 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:56,571 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:34:56,572 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:56,577 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:57,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:34:57,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:34:58,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:01,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:01,475 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:01,475 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:01,480 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:02,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:03,449 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:04,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:05,933 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:35:06,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:06,084 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:06,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:06,089 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:06,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:07,454 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:10,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:10,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:35:10,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:35:10,961 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:35:11,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:11,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:11,258 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:11,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:11,457 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:13,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:15,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:15,162 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:15,163 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:15,163 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:15,168 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:15,461 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:17,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:20,169 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:20,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:20,240 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:20,240 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:20,245 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:20,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:21,469 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:24,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:24,242 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:24,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:24,247 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:24,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:25,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:25,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:25,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:35:25,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:35:25,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:35:31,179 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:32,709 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:32,710 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:32,710 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:32,715 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:33,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:35,481 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:35,932 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:35:36,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:36,569 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:36,569 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:36,569 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:36,574 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:37,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:37,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:40,609 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:40,609 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:40,610 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:40,615 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:40,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:35:40,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:35:40,961 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:35:41,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:41,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:42,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:45,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:45,288 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:45,288 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:45,293 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:45,491 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:47,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:47,493 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:50,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:50,562 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:50,562 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:50,567 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:51,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:51,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:52,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:54,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:54,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:54,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:54,517 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:55,501 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:55,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:35:55,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:35:55,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:35:55,961 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:35:58,198 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:58,527 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:58,528 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:35:58,528 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:58,533 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:59,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:35:59,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:02,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:02,874 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:02,875 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:02,879 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:03,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:03,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:05,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:05,937 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:36:07,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:07,320 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:07,321 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:07,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:07,514 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:09,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:09,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:10,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:36:10,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:36:10,961 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:36:11,884 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:11,884 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:11,885 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:11,890 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:12,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:13,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:14,890 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:17,485 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:17,485 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:17,486 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:17,491 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:17,524 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:19,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:20,491 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:25,492 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:25,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:36:25,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:36:25,961 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:36:26,721 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:26,722 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:26,722 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:26,727 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:27,532 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:29,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:30,728 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:31,067 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:31,067 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:31,068 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:31,073 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:31,536 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:33,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:35,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:35,581 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:35,581 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:35,586 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:35,936 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:36:35,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:36,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:37,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:39,771 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:39,772 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:39,772 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:39,777 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:40,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:40,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:36:40,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:36:40,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:36:41,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:41,548 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:46,182 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:46,183 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:46,183 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:46,188 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:46,552 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:47,189 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:47,555 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:50,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:50,888 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:50,888 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:50,894 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:51,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:52,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:53,560 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:55,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:55,161 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:36:55,161 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:55,166 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:55,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:36:55,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:36:55,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:36:55,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:36:57,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:36:58,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:00,573 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:00,574 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:00,574 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:00,579 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:01,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:01,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:03,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:05,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:05,646 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:05,646 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:05,651 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:05,938 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:37:06,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:07,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:08,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:10,333 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:10,333 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:10,334 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:10,340 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:10,577 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:10,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:37:10,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:37:10,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:37:11,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:14,232 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:14,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:14,346 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:14,347 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:14,351 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:14,582 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:15,585 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:19,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:19,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:19,569 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:19,569 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:19,574 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:19,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:21,590 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:23,792 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:23,793 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:23,793 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:23,798 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:24,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:24,799 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:25,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:25,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:37:25,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:37:25,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:37:29,436 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:29,437 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:29,437 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:29,442 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:29,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:30,443 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:31,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:35,443 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:35,941 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:37:36,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:36,329 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:36,329 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:36,334 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:36,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:37,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:40,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:40,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:40,469 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:40,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:40,474 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:40,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:40,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:37:40,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:37:40,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:37:41,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:44,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:44,931 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:44,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:44,936 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:45,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:45,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:47,616 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:49,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:49,724 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:49,724 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:49,729 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:50,618 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:51,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:51,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:54,594 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:54,595 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:54,595 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:54,600 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:54,622 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:55,624 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:37:55,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:37:55,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:37:55,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:37:57,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:58,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:58,287 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:37:58,288 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:58,292 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:58,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:37:59,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:02,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:03,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:03,960 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:03,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:03,965 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:04,632 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:05,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:05,943 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:38:07,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:08,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:08,160 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:08,161 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:08,165 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:08,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:09,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:10,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:38:10,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:38:10,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:38:13,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:13,007 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:13,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:13,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:13,013 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:13,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:15,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:16,876 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:16,877 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:16,877 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:16,882 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:17,647 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:18,883 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:19,651 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:22,916 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:22,917 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:22,917 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:22,922 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:23,653 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:23,923 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:25,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:25,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:38:25,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:38:25,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:38:26,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:26,763 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:26,763 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:26,768 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:27,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:27,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:29,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:32,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:32,260 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:32,260 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:32,265 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:32,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:33,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:35,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:35,945 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:38:36,455 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:36,456 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:36,456 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:36,461 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:36,666 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:37,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:39,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:39,815 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:39,815 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:39,820 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:40,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:40,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:40,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:38:40,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:38:40,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:38:41,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:43,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:43,159 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:43,159 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:43,165 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:43,672 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:45,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:46,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:48,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:48,013 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:48,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:48,019 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:48,678 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:49,680 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:38:52,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:55,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:55,791 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:38:55,792 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:55,796 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:55,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:38:55,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:38:55,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:38:56,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:38:57,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:57,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:00,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:00,097 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:00,097 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:00,102 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:00,689 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:01,691 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:03,103 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:04,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:04,060 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:04,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:04,066 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:04,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:05,697 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:05,947 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:39:08,436 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:08,437 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:08,437 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:08,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:08,442 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:08,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:09,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:10,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:39:10,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:39:10,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:39:12,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:12,576 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:12,576 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:12,581 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:12,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:13,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:13,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:16,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:16,569 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:16,569 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:16,574 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:16,706 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:17,708 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:19,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:24,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:25,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:39:25,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:39:25,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:39:26,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:26,955 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:26,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:26,960 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:27,714 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:29,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:29,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:30,780 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:30,781 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:30,781 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:30,786 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:31,720 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:31,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:35,787 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:35,952 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:39:36,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:36,307 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:36,307 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:36,312 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:36,723 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:37,726 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:40,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:39:40,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:39:40,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:39:41,191 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:41,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:41,945 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:41,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:41,950 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:42,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:43,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:46,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:46,946 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:46,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:46,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:46,952 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:47,734 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:49,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:51,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:52,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:52,742 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:52,742 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:52,747 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:53,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:53,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:39:55,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:39:55,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:39:55,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:39:57,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:57,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:57,835 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:39:57,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:57,840 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:58,745 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:39:59,747 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:01,760 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:01,761 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:01,761 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:01,766 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:02,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:02,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:03,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:05,951 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:40:07,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:07,055 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:07,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:07,060 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:07,755 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:08,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:09,758 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:10,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:40:10,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:40:10,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:40:11,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:11,240 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:11,240 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:11,245 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:11,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:13,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:13,763 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:16,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:16,014 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:16,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:16,019 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:16,765 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:17,767 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:19,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:20,734 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:20,734 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:20,735 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:20,740 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:20,769 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:21,772 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:24,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:25,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:40:25,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:40:25,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:40:26,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:26,186 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:26,186 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:26,191 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:26,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:27,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:30,192 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:34,498 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:34,498 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:34,499 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:34,503 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:34,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:35,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:35,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:35,956 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:40:38,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:38,623 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:38,624 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:38,629 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:38,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:39,787 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:40,630 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:40,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:40:40,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:40:40,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:40:46,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:47,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:47,618 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:47,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:47,623 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:47,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:49,795 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:51,624 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:52,395 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:52,396 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:52,397 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:52,401 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:52,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:53,800 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:40:55,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:40:55,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:40:55,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:40:57,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:57,024 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:40:57,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:57,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:57,030 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:57,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:40:59,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:01,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:01,273 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:01,273 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:01,279 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:01,807 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:02,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:03,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:05,955 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:41:06,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:06,225 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:06,225 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:06,230 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:06,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:07,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:08,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:10,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:41:10,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:41:10,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:41:13,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:14,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:14,052 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:14,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:14,057 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:14,820 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:15,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:18,412 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:18,413 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:18,413 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:18,413 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:18,418 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:18,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:19,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:23,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:25,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:41:25,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:41:25,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:41:26,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:26,850 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:26,850 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:26,855 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:27,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:27,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:28,856 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:30,906 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:30,907 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:30,907 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:30,912 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:31,839 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:31,839 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:33,913 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:34,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:34,869 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:34,869 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:34,874 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:35,843 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:35,843 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:35,957 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:41:38,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:40,919 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:40,920 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:40,920 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:40,925 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:40,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:41:40,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:41:40,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:41:41,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:41,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:44,184 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:45,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:45,954 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:45,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:45,959 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:46,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:47,856 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:49,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:51,333 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:51,334 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:51,334 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:51,339 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:51,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:53,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:55,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:55,095 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:55,095 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:55,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:55,101 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:55,862 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:41:55,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:41:55,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:41:55,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:41:57,869 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:41:59,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:59,297 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:41:59,297 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:59,302 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:59,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:00,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:01,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:03,851 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:03,851 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:03,852 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:03,857 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:03,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:05,857 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:05,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:05,959 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:42:08,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:08,832 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:08,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:08,837 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:08,879 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:09,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:10,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:42:10,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:42:10,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:42:11,182 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:14,161 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:14,162 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:14,162 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:14,167 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:14,885 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:15,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:17,168 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:18,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:18,323 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:18,323 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:18,328 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:18,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:19,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:22,071 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:22,072 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:22,073 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:22,077 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:22,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:23,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:23,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:25,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:42:25,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:42:25,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:42:26,636 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:26,637 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:26,637 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:26,642 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:26,898 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:27,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:28,643 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:31,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:31,076 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:31,076 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:31,081 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:31,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:31,906 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:34,082 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:35,498 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:35,499 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:35,499 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:35,504 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:35,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:35,961 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:42:37,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:39,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:39,428 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:39,429 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:39,429 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:39,434 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:39,912 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:40,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:42:40,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:42:40,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:42:41,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:43,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:43,962 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:43,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:43,967 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:44,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:44,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:45,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:47,519 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:47,520 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:47,520 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:47,525 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:47,922 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:49,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:50,526 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:51,145 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:51,146 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:51,147 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:51,152 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:51,927 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:53,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:54,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:54,903 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:42:54,903 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:54,908 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:54,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:42:55,908 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:55,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:42:55,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:42:55,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:42:55,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:43:00,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:00,291 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:00,291 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:00,296 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:00,934 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:01,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:01,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:04,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:04,003 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:04,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:04,008 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:04,940 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:05,943 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:05,966 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:43:06,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:08,165 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:08,166 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:08,166 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:08,171 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:08,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:09,947 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:10,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:43:10,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:43:10,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:43:12,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:12,358 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:12,359 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:12,359 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:12,364 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:12,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:13,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:15,911 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:15,911 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:15,912 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:15,917 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:15,953 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:17,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:17,956 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:20,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:20,171 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:20,172 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:20,177 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:20,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:21,961 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:23,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:25,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:43:25,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:43:25,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:43:26,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:26,630 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:26,630 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:26,636 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:26,966 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:27,968 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:28,637 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:30,415 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:30,416 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:30,416 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:30,421 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:30,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:31,973 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:34,128 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:34,129 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:34,130 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:34,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:34,135 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:34,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:35,965 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:43:35,977 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:38,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:38,535 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:38,535 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:38,540 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:38,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:39,540 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:39,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:40,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:43:40,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:43:40,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:43:42,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:42,483 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:42,483 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:42,488 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:42,983 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:43,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:45,489 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:46,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:46,262 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:46,262 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:46,267 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:46,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:47,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:49,791 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:49,792 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:49,792 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:49,797 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:49,993 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:50,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:51,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:54,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:54,210 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:54,210 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:54,215 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:54,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:43:55,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:43:55,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:43:55,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:43:55,999 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:43:56,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:58,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:58,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:43:58,523 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:58,528 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:59,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:00,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:01,529 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:05,888 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:05,889 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:05,889 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:05,894 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:05,970 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:44:06,008 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:06,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:08,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:10,118 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:10,119 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:10,119 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:10,126 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:10,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:44:10,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:44:10,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:44:11,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:12,016 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:12,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:14,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:14,420 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:14,421 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:14,426 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:15,018 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:16,020 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:17,426 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:20,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:20,227 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:20,227 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:20,232 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:21,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:22,026 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:23,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:24,636 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:24,637 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:24,638 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:24,643 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:25,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:25,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:44:25,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:44:25,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:44:26,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:28,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:28,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:28,865 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:28,866 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:28,871 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:29,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:30,036 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:32,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:32,858 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:32,859 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:32,864 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:33,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:33,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:34,040 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:35,970 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:44:38,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:38,578 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:38,578 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:38,583 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:39,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:39,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:40,045 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:40,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:44:40,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:44:40,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:44:43,804 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:43,805 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:43,805 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:43,810 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:44,048 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:44,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:46,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:47,676 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:47,677 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:47,677 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:47,683 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:48,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:50,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:50,684 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:53,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:53,064 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:44:53,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:53,069 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:54,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:44:54,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:44:55,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:44:55,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:44:55,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:44:56,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:01,196 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:01,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:01,366 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:01,367 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:01,372 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:02,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:04,071 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:05,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:05,686 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:05,686 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:05,691 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:05,974 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:45:06,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:06,976 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:08,075 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:09,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:09,428 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:09,428 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:09,433 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:10,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:10,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:45:10,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:45:10,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:45:12,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:12,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:13,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:13,657 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:13,658 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:13,662 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:14,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:16,084 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:17,663 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:18,107 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:18,108 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:18,108 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:18,113 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:19,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:20,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:22,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:22,690 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:22,690 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:22,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:22,695 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:23,091 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:24,093 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:25,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:45:25,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:45:25,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:45:26,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:26,623 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:26,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:26,628 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:27,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:28,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:28,628 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:31,462 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:31,463 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:31,463 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:31,468 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:32,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:34,103 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:34,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:35,974 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:45:36,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:36,814 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:36,814 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:36,819 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:37,105 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:38,108 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:39,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:40,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:45:40,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:45:40,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:45:41,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:41,263 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:41,263 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:41,268 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:42,113 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:42,113 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:44,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:44,972 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:44,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:44,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:44,978 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:45,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:46,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:48,179 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:48,179 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:48,180 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:48,184 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:49,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:50,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:50,185 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:51,895 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:51,896 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:51,896 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:51,901 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:52,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:54,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:45:55,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:55,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:45:55,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:45:55,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:45:56,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:56,561 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:45:56,562 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:56,567 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:57,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:45:58,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:00,693 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:00,694 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:00,694 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:00,699 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:01,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:01,700 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:02,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:04,405 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:04,406 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:04,406 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:04,411 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:05,138 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:05,982 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:46:06,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:06,983 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:09,352 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:09,353 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:09,353 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:09,358 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:10,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:10,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:10,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:46:10,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:46:10,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:46:12,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:15,483 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:15,484 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:15,484 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:15,489 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:16,149 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:17,525 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:18,152 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:20,533 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:20,534 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:20,534 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:20,539 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:21,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:22,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:22,540 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:25,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:46:25,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:46:25,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:46:26,492 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:26,493 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:26,493 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:26,498 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:27,160 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:28,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:28,537 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:31,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:31,564 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:31,564 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:31,569 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:32,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:33,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:34,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:35,862 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:35,862 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:35,863 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:35,868 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:35,978 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:46:36,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:38,173 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:38,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:39,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:39,646 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:39,646 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:39,652 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:40,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:40,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:46:40,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:46:40,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:46:42,177 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:44,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:44,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:44,466 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:44,466 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:44,471 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:45,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:46,183 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:49,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:53,113 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:53,114 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:53,114 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:53,119 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:53,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:46:54,189 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:46:55,120 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:55,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:46:55,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:46:55,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:46:58,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:58,032 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:46:58,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:58,037 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:58,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:00,193 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:01,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:02,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:02,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:02,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:02,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:03,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:04,198 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:05,980 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:47:06,982 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:08,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:08,031 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:08,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:08,037 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:08,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:10,204 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:10,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:47:10,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:47:10,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:47:12,185 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:13,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:13,267 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:13,267 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:13,272 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:14,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:14,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:17,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:17,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:17,620 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:17,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:17,625 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:18,211 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:20,215 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:22,626 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:23,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:23,142 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:23,142 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:23,147 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:23,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:24,219 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:25,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:47:25,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:47:25,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:47:28,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:28,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:28,598 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:28,598 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:28,603 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:29,222 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:30,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:32,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:32,994 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:32,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:33,000 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:33,226 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:34,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:34,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:35,982 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:47:37,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:37,770 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:37,770 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:37,775 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:38,232 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:39,776 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:40,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:40,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:47:40,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:47:40,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:47:45,240 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:46,181 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:46,181 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:46,182 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:46,187 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:46,241 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:48,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:49,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:49,968 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:49,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:49,973 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:50,247 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:50,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:52,252 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:47:55,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:55,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:47:55,259 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:55,264 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:55,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:47:55,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:47:55,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:47:56,211 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:56,255 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:47:56,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:01,211 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:02,155 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:02,156 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:02,156 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:02,161 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:02,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:04,270 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:05,984 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:48:06,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:08,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:08,261 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:08,262 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:08,266 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:08,273 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:10,278 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:10,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:48:10,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:48:10,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:48:12,198 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:13,327 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:13,327 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:13,328 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:13,332 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:14,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:14,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:17,369 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:18,912 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:18,913 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:18,913 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:18,918 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:19,289 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:20,292 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:22,394 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:22,395 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:22,395 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:22,395 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:22,400 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:23,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:24,298 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:25,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:48:25,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:48:25,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:48:26,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:26,949 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:26,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:26,954 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:27,301 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:27,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:28,304 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:32,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:32,867 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:32,867 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:32,872 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:33,310 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:33,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:34,313 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:35,990 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:48:36,817 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:36,818 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:36,818 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:36,823 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:37,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:38,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:39,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:40,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:48:40,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:48:40,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:48:41,905 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:41,905 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:41,906 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:41,911 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:42,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:44,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:44,911 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:46,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:46,032 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:46,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:46,037 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:46,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:48,333 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:49,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:49,836 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:49,836 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:49,841 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:50,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:50,841 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:52,339 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:48:55,842 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:55,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:48:55,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:48:55,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:48:56,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:56,264 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:48:56,264 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:56,269 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:56,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:48:58,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:01,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:04,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:04,608 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:04,609 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:04,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:05,355 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:05,988 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:49:06,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:06,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:08,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:08,159 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:08,160 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:08,165 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:08,359 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:10,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:10,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:49:10,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:49:10,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:49:11,799 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:11,800 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:11,800 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:11,806 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:12,366 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:12,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:14,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:16,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:16,213 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:16,214 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:16,219 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:16,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:18,220 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:18,376 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:21,442 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:21,443 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:21,443 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:21,448 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:22,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:22,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:23,449 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:25,384 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:25,385 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:25,385 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:25,390 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:25,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:49:25,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:49:25,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:49:26,388 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:26,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:29,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:30,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:30,582 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:30,582 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:30,587 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:31,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:32,397 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:34,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:35,400 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:35,401 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:35,402 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:35,407 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:35,994 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:49:36,403 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:36,403 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:39,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:39,653 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:39,654 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:39,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:39,659 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:40,406 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:40,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:49:40,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:49:40,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:49:42,411 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:43,524 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:43,524 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:43,525 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:43,530 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:44,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:44,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:45,565 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:48,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:48,060 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:48,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:48,065 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:48,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:50,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:51,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:53,079 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:53,079 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:53,080 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:53,084 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:53,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:54,428 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:49:55,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:49:55,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:49:55,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:49:56,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:57,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:57,056 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:49:57,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:57,061 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:57,431 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:49:58,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:02,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:02,728 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:02,728 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:02,729 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:02,734 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:03,440 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:04,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:05,993 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:50:06,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:06,483 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:06,483 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:06,488 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:07,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:07,489 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:08,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:10,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:10,505 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:10,505 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:10,510 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:10,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:50:10,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:50:10,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:50:11,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:12,454 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:13,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:15,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:15,485 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:15,485 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:15,490 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:16,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:16,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:18,525 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:19,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:19,533 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:19,533 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:19,539 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:20,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:20,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:23,539 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:23,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:23,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:23,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:23,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:24,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:25,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:50:25,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:50:25,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:50:26,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:29,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:29,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:29,922 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:29,923 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:29,928 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:30,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:32,483 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:33,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:33,734 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:33,734 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:33,739 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:34,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:34,740 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:35,995 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:50:36,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:37,175 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:37,176 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:37,176 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:37,181 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:37,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:38,491 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:40,182 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:40,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:50:40,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:50:40,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:50:41,416 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:41,417 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:41,417 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:41,422 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:41,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:42,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:45,423 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:47,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:47,308 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:47,308 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:47,313 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:47,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:48,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:51,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:51,056 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:51,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:51,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:51,061 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:51,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:52,512 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:55,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:55,415 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:55,416 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:55,421 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:55,515 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:50:55,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:50:55,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:50:55,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:50:56,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:56,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:50:59,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:59,168 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:50:59,168 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:59,173 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:59,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:00,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:01,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:03,376 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:03,377 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:03,377 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:03,382 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:03,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:04,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:05,997 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:51:06,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:09,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:09,434 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:09,434 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:09,439 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:09,536 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:10,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:10,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:51:10,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:51:10,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:51:12,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:13,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:13,291 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:13,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:13,297 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:13,542 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:14,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:18,333 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:19,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:19,056 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:19,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:19,061 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:19,551 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:20,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:23,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:23,384 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:23,384 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:23,384 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:23,389 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:23,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:24,559 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:25,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:51:25,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:51:25,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:51:27,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:27,290 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:27,290 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:27,295 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:27,563 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:28,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:29,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:31,143 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:31,144 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:31,144 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:31,149 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:31,568 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:32,571 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:35,150 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:35,999 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:51:39,324 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:39,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:39,325 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:39,330 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:39,579 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:40,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:40,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:40,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:51:40,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:51:40,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:51:42,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:42,598 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:42,598 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:42,603 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:43,585 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:44,588 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:45,603 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:48,851 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:48,852 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:48,852 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:48,857 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:49,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:50,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:50,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:53,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:53,020 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:53,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:53,025 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:53,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:51:54,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:51:55,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:51:55,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:51:55,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:51:56,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:58,535 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:58,535 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:51:58,536 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:58,540 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:58,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:00,611 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:01,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:04,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:04,747 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:04,747 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:04,752 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:05,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:06,001 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:52:06,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:07,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:08,601 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:08,602 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:08,602 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:08,608 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:08,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:10,625 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:10,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:52:10,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:52:10,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:52:12,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:13,128 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:13,128 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:13,129 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:13,134 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:13,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:14,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:16,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:16,964 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:16,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:16,970 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:17,634 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:17,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:18,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:22,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:25,431 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:25,431 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:25,432 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:25,437 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:25,645 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:25,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:52:25,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:52:25,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:52:26,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:28,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:31,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:31,270 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:31,271 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:31,276 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:31,654 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:32,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:33,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:35,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:35,682 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:35,683 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:35,688 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:36,003 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:52:36,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:36,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:39,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:40,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:52:40,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:52:40,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:52:42,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:42,420 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:42,420 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:42,425 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:42,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:44,426 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:44,674 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:48,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:48,172 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:48,172 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:48,177 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:48,678 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:50,178 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:50,683 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:52,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:52,098 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:52,098 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:52,103 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:52,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:54,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:52:55,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:52:55,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:52:55,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:52:56,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:56,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:56,316 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:52:56,316 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:56,321 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:56,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:52:58,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:00,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:00,958 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:00,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:00,964 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:01,697 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:01,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:02,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:06,009 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:53:06,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:06,107 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:06,107 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:06,112 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:06,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:07,113 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:08,708 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:10,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:10,284 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:10,285 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:10,290 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:10,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:10,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:53:10,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:53:10,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:53:12,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:12,714 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:17,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:18,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:18,794 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:18,794 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:18,798 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:19,719 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:20,722 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:22,837 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:23,115 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:23,116 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:23,117 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:23,122 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:23,724 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:24,727 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:25,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:53:25,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:53:25,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:53:28,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:28,004 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:28,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:28,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:28,009 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:28,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:30,732 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:31,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:31,835 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:31,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:31,840 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:32,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:32,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:33,841 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:35,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:35,989 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:35,989 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:35,994 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:36,008 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:53:36,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:38,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:39,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:40,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:40,807 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:40,807 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:40,812 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:40,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:53:40,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:53:40,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:53:41,743 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:42,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:44,184 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:46,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:46,497 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:46,497 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:46,502 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:46,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:48,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:49,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:53,533 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:53,533 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:53,534 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:53,539 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:53,755 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:53:54,539 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:54,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:53:55,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:53:55,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:53:55,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:53:59,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:59,572 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:53:59,572 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:59,572 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:59,577 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:59,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:00,763 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:04,578 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:04,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:05,000 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:05,000 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:05,005 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:05,767 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:06,013 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:54:06,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:08,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:08,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:08,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:08,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:09,772 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:09,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:10,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:10,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:54:10,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:54:10,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:54:12,847 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:12,848 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:12,848 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:12,853 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:13,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:14,777 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:15,854 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:18,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:18,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:18,279 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:18,284 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:18,780 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:20,783 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:21,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:22,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:22,731 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:22,731 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:22,737 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:22,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:24,787 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:25,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:54:25,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:54:25,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:54:26,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:26,948 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:26,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:26,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:26,953 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:27,789 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:28,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:30,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:30,968 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:30,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:30,974 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:31,793 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:31,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:32,796 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:36,012 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:54:36,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:36,347 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:36,347 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:36,352 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:36,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:37,352 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:38,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:40,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:54:40,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:54:40,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:54:42,324 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:42,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:42,325 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:42,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:42,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:43,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:44,808 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:46,564 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:46,564 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:46,565 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:46,570 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:46,810 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:48,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:48,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:50,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:50,696 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:50,696 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:50,701 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:50,815 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:52,818 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:53,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:54,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:54,256 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:54,257 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:54,262 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:54,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:55,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:54:55,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:54:55,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:54:56,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:54:58,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:58,601 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:54:58,601 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:58,606 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:58,824 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:54:59,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:00,827 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:02,357 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:02,358 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:02,358 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:02,363 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:02,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:04,831 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:05,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:06,014 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:55:06,492 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:06,493 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:06,494 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:06,499 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:06,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:08,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:10,500 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:10,675 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:10,676 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:10,676 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:10,682 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:10,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:10,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:55:10,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:55:10,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:55:12,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:14,446 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:14,446 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:14,447 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:14,451 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:14,841 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:16,489 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:16,845 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:18,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:18,279 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:18,279 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:18,284 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:18,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:20,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:22,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:23,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:23,253 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:23,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:23,258 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:23,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:24,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:25,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:55:25,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:55:25,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:55:28,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:28,724 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:28,725 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:28,725 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:28,730 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:28,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:30,860 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:32,609 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:32,610 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:32,610 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:32,615 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:32,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:33,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:34,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:36,016 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:55:37,803 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:37,804 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:37,804 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:37,809 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:37,866 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:38,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:38,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:40,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:55:40,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:55:40,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:55:41,771 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:41,772 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:41,773 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:41,778 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:41,868 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:42,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:44,778 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:49,779 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:50,411 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:50,412 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:50,412 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:50,418 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:50,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:52,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:55,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:55,613 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:55,613 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:55,614 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:55,619 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:55,883 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:55:55,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:55:55,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:55:55,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:55:56,885 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:55:59,987 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:59,988 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:55:59,988 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:59,993 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:00,888 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:00,888 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:01,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:04,305 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:04,306 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:04,306 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:04,311 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:04,891 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:06,018 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:56:06,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:07,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:10,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:10,103 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:10,103 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:10,108 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:10,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:10,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:56:10,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:56:10,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:56:12,199 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:12,900 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:14,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:14,084 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:14,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:14,089 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:14,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:16,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:18,125 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:22,544 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:22,545 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:22,545 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:22,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:22,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:23,551 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:24,912 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:25,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:56:25,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:56:25,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:56:26,220 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:26,280 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:26,280 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:26,285 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:26,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:28,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:29,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:30,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:30,368 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:30,369 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:30,374 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:30,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:32,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:34,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:34,486 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:34,490 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:34,490 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:34,494 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:34,920 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:36,020 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:56:36,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:38,342 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:38,342 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:38,343 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:38,348 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:38,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:40,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:40,927 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:40,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:56:40,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:56:40,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:56:43,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:43,690 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:43,690 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:43,695 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:43,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:44,932 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:45,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:49,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:49,292 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:49,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:49,297 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:49,936 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:50,936 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:51,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:54,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:54,827 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:56:54,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:54,833 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:54,939 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:56:55,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:56:55,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:56:55,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:56:56,942 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:56:57,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:01,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:01,477 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:01,477 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:01,482 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:01,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:02,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:02,948 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:06,022 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:57:06,161 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:06,161 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:06,162 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:06,167 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:06,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:08,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:08,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:10,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:57:10,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:57:10,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:57:11,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:11,206 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:11,207 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:11,212 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:11,956 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:12,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:13,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:16,632 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:16,633 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:16,633 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:16,638 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:16,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:18,638 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:18,965 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:20,587 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:20,588 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:20,588 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:20,593 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:20,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:22,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:24,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:24,280 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:24,280 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:24,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:24,285 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:24,971 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:25,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:57:25,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:57:25,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:57:26,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:28,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:28,234 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:28,235 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:28,240 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:28,976 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:30,240 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:30,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:33,379 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:33,380 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:33,381 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:33,385 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:33,981 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:34,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:35,386 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:36,028 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:57:37,837 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:37,838 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:37,838 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:37,843 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:37,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:38,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:40,844 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:40,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:57:40,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:57:40,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:57:43,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:43,846 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:43,846 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:43,851 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:43,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:44,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:45,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:48,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:48,063 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:48,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:48,068 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:48,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:48,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:51,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:52,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:52,686 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:52,686 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:52,691 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:53,000 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:55,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:57:55,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:57:55,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:57:55,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:57:56,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:56,863 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:56,864 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:57:56,865 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:56,870 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:57,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:57:59,008 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:01,105 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:01,106 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:01,106 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:01,111 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:02,010 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:02,112 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:03,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:05,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:05,236 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:05,237 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:05,242 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:06,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:06,027 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:58:07,018 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:08,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:09,098 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:09,099 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:09,100 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:09,105 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:10,020 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:10,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:58:10,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:58:10,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:58:11,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:13,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:14,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:14,093 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:14,094 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:14,099 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:15,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:15,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:18,128 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:18,129 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:18,130 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:18,134 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:19,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:19,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:19,135 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:22,443 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:22,444 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:22,444 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:22,450 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:23,034 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:24,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:25,038 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:25,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:58:25,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:58:25,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:58:27,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:27,177 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:27,177 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:27,182 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:28,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:29,042 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:30,183 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:31,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:31,742 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:31,742 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:31,747 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:32,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:33,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:35,667 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:35,668 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:35,669 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:35,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:35,674 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:36,032 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:58:36,048 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:37,051 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:39,712 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:39,713 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:39,714 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:39,719 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:40,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:40,719 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:40,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:58:40,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:58:40,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:58:41,053 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:44,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:44,291 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:44,291 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:44,296 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:45,055 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:46,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:47,060 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:51,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:52,418 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:52,419 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:52,419 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:52,424 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:53,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:55,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:58:55,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:58:55,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:58:55,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:58:57,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:57,266 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:58:57,267 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:57,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:57,271 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:58,068 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:58:59,071 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:02,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:02,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:02,646 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:02,646 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:02,651 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:03,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:05,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:06,031 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:59:07,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:07,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:07,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:07,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:07,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:08,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:09,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:10,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:59:10,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:59:10,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:59:12,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:12,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:12,462 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:12,467 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:13,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:13,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:15,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:16,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:16,670 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:16,670 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:16,675 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:17,091 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:18,676 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:19,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:23,387 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:23,388 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:23,388 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:23,393 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:24,097 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:24,394 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:25,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:25,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:59:25,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:59:25,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:59:26,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:26,903 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:26,904 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:26,909 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:27,101 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:29,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:29,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:33,183 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:33,183 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:33,184 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:33,189 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:34,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:35,110 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:35,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:36,036 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 13:59:37,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:37,791 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:37,791 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:37,796 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:38,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:39,115 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:40,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:40,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:59:40,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:59:40,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:59:41,979 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:41,980 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:41,980 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:41,986 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:42,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:43,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:45,986 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:46,187 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:46,188 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:46,188 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:46,193 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:47,125 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:47,125 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:51,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:51,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:51,261 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:51,261 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:51,266 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:52,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:53,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 13:59:55,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:59:55,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:59:55,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:59:56,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:56,700 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:56,701 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 13:59:56,702 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:56,707 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:57,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 13:59:59,136 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:01,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:01,002 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:01,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:01,008 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:01,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:02,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:03,141 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:05,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:05,326 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:05,326 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:05,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:06,035 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:00:06,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:07,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:07,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:10,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:00:10,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:00:10,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:00:11,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:11,264 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:11,265 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:11,269 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:12,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:12,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:13,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:16,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:16,316 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:16,316 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:16,321 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:17,153 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:17,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:19,156 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:21,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:21,264 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:21,264 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:21,270 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:22,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:23,161 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:23,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:25,199 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:25,200 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:25,200 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:25,206 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:25,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:00:25,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:00:25,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:00:26,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:27,166 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:29,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:30,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:30,283 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:30,283 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:30,289 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:31,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:31,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:34,174 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:34,174 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:34,175 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:34,180 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:35,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:35,175 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:35,180 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:36,037 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:00:37,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:37,459 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:37,459 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:37,464 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:38,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:39,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:40,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:40,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:00:40,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:00:40,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:00:42,311 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:42,312 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:42,312 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:42,317 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:43,184 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:43,184 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:46,318 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:48,979 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:48,980 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:48,980 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:48,985 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:49,188 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:51,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:51,986 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:52,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:52,968 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:52,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:52,974 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:53,193 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:55,196 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:55,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:00:55,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:00:55,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:00:56,525 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:56,526 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:56,527 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:56,532 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:57,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:00:57,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:59,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:00:59,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:59,961 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:00:59,961 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:59,966 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:00,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:01,204 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:02,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:04,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:04,043 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:04,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:04,048 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:04,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:05,208 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:06,039 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:01:07,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:07,730 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:07,730 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:07,735 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:08,210 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:08,736 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:09,212 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:10,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:01:10,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:01:10,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:01:14,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:16,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:16,726 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:16,726 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:16,731 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:17,218 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:19,221 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:19,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:21,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:21,547 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:21,547 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:21,552 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:22,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:23,226 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:25,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:25,064 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:25,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:25,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:25,069 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:25,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:25,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:01:25,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:01:25,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:01:27,231 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:29,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:29,647 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:29,648 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:29,653 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:30,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:30,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:31,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:33,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:33,583 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:33,584 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:33,589 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:34,237 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:35,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:36,041 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:01:36,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:37,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:37,328 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:37,329 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:37,334 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:38,240 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:39,242 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:40,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:01:40,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:01:40,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:01:41,181 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:41,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:41,648 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:41,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:41,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:42,244 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:43,247 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:46,088 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:46,089 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:46,089 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:46,095 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:46,249 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:47,095 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:47,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:50,399 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:50,400 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:50,400 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:50,405 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:51,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:51,258 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:52,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:55,608 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:55,609 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:55,609 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:55,615 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:55,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:01:55,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:01:55,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:01:56,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:01:57,263 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:01:58,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:59,785 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:59,786 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:01:59,786 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:59,792 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:00,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:01,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:03,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:06,047 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:02:06,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:06,230 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:06,230 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:06,235 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:06,272 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:07,274 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:09,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:10,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:10,107 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:10,107 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:10,113 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:10,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:10,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:02:10,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:02:10,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:02:11,279 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:13,903 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:13,903 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:13,904 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:13,909 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:14,281 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:14,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:15,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:19,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:21,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:21,869 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:21,869 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:21,874 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:22,289 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:23,291 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:25,875 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:25,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:02:25,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:02:25,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:02:26,401 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:26,402 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:26,402 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:26,407 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:27,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:27,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:31,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:34,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:34,651 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:34,652 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:34,657 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:35,301 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:36,046 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:02:37,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:37,304 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:38,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:38,790 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:38,791 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:38,796 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:39,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:40,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:02:40,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:02:40,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:02:41,308 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:42,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:44,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:44,130 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:44,130 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:44,135 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:44,310 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:45,312 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:47,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:47,546 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:47,547 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:47,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:47,552 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:48,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:49,318 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:52,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:52,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:52,578 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:52,578 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:52,583 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:53,321 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:55,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:02:55,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:02:55,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:02:55,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:02:58,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:58,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:58,280 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:02:58,281 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:58,286 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:58,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:02:59,328 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:02,768 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:02,768 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:02,769 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:02,774 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:03,331 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:03,774 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:05,335 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:06,050 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:03:09,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:09,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:09,312 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:09,313 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:09,318 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:09,337 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:10,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:03:10,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:03:10,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:03:11,341 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:14,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:14,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:14,535 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:14,536 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:14,541 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:15,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:17,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:18,637 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:18,638 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:18,638 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:18,643 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:19,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:19,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:21,352 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:24,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:25,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:03:25,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:03:25,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:03:26,352 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:26,353 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:26,353 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:26,358 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:27,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:27,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:30,359 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:34,839 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:34,840 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:34,840 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:34,845 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:35,362 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:35,846 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:36,050 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:03:37,366 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:39,085 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:39,086 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:39,086 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:39,091 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:39,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:40,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:03:40,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:03:40,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:03:41,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:41,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:43,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:43,862 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:43,863 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:43,868 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:44,372 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:45,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:46,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:48,079 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:48,079 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:48,080 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:48,085 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:48,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:49,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:52,086 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:52,624 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:52,625 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:52,626 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:52,631 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:53,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:55,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:03:55,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:03:55,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:03:55,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:03:57,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:57,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:57,482 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:03:57,482 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:57,488 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:58,388 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:03:59,391 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:02,134 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:02,135 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:02,135 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:02,141 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:02,393 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:03,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:03,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:05,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:05,887 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:05,887 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:05,893 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:06,055 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:04:06,399 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:07,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:09,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:09,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:09,737 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:09,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:09,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:10,403 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:10,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:04:10,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:04:10,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:04:11,406 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:14,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:15,313 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:15,314 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:15,314 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:15,319 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:15,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:17,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:18,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:18,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:18,853 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:18,858 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:19,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:19,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:21,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:22,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:22,821 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:22,821 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:22,826 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:23,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:25,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:25,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:25,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:04:25,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:04:25,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:04:27,640 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:27,641 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:27,641 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:27,646 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:28,423 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:29,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:31,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:31,382 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:31,382 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:31,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:31,388 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:31,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:33,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:36,054 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:04:36,480 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:36,481 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:36,482 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:36,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:36,487 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:37,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:37,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:40,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:04:40,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:04:40,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:04:42,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:43,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:43,098 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:43,098 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:43,103 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:43,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:45,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:47,343 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:47,344 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:47,344 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:47,344 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:47,349 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:47,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:49,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:52,165 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:52,165 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:52,166 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:52,171 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:52,449 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:53,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:53,452 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:04:55,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:04:55,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:04:55,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:04:57,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:57,702 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:04:57,702 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:57,707 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:58,454 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:04:58,708 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:59,457 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:03,708 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:04,395 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:04,396 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:04,396 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:04,401 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:04,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:05,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:06,056 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:05:09,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:09,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:09,288 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:09,288 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:09,293 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:09,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:10,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:05:10,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:05:10,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:05:11,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:14,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:17,643 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:17,644 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:17,644 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:17,649 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:18,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:19,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:19,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:23,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:23,502 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:23,502 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:23,507 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:24,480 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:25,483 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:25,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:25,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:05:25,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:05:25,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:05:29,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:29,226 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:29,226 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:29,231 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:29,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:31,232 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:31,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:33,072 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:33,073 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:33,074 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:33,079 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:33,490 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:35,493 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:36,058 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:05:37,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:37,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:37,284 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:37,285 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:37,290 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:37,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:39,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:40,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:05:40,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:05:40,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:05:41,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:41,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:41,296 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:41,301 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:41,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:42,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:43,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:45,978 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:45,979 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:45,979 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:45,984 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:46,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:47,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:47,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:49,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:49,930 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:49,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:49,936 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:50,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:51,512 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:53,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:53,535 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:53,535 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:53,535 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:53,540 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:54,514 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:55,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:05:55,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:05:55,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:05:55,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:05:57,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:57,118 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:05:57,118 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:57,123 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:57,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:05:59,124 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:59,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:04,124 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:05,131 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:05,132 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:05,132 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:05,138 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:05,525 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:06,060 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:06:07,528 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:09,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:09,073 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:09,074 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:09,079 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:09,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:10,080 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:10,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:06:10,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:06:10,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:06:11,533 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:13,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:13,076 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:13,077 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:13,082 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:13,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:15,082 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:15,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:16,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:16,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:16,699 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:16,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:17,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:17,542 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:20,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:20,548 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:20,549 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:20,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:20,554 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:21,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:21,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:25,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:25,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:06:25,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:06:25,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:06:26,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:26,181 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:26,181 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:26,186 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:26,548 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:27,551 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:29,818 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:29,819 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:29,819 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:29,824 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:30,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:30,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:31,555 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:35,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:36,066 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:06:38,189 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:38,190 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:38,190 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:38,195 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:38,560 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:39,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:40,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:06:40,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:06:40,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:06:41,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:42,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:42,015 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:42,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:42,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:42,564 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:43,566 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:45,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:45,582 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:45,582 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:45,588 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:46,568 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:46,588 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:47,572 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:49,336 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:49,336 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:49,337 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:49,342 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:49,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:51,577 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:06:52,343 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:55,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:06:55,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:06:55,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:06:57,413 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:57,413 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:06:57,414 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:57,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:57,419 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:57,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:06:59,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:02,420 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:05,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:05,957 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:05,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:05,962 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:06,064 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:07:06,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:07,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:08,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:09,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:09,778 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:09,778 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:09,783 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:10,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:10,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:07:10,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:07:10,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:07:11,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:13,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:14,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:14,260 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:14,260 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:14,265 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:14,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:15,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:17,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:17,958 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:17,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:17,963 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:18,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:18,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:19,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:21,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:21,835 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:21,836 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:21,841 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:22,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:23,611 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:24,842 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:25,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:07:25,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:07:25,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:07:28,240 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:28,241 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:28,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:28,246 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:28,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:29,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:30,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:32,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:32,277 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:32,277 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:32,282 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:32,619 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:33,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:35,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:36,070 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:07:36,193 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:36,194 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:36,194 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:36,199 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:36,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:37,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:40,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:07:40,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:07:40,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:07:41,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:41,174 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:41,174 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:41,174 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:41,179 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:41,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:43,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:46,180 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:46,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:46,329 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:46,329 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:46,334 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:46,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:47,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:50,232 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:50,233 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:50,234 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:50,239 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:50,638 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:51,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:51,641 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:07:55,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:07:55,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:07:55,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:07:57,227 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:58,632 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:58,633 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:07:58,633 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:58,638 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:58,645 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:07:59,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:02,447 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:02,448 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:02,448 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:02,449 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:02,454 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:02,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:03,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:06,069 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:08:07,740 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:07,741 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:07,741 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:07,742 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:07,747 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:08,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:09,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:10,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:08:10,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:08:10,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:08:11,908 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:11,909 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:11,910 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:11,915 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:12,661 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:12,916 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:13,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:16,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:16,793 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:16,794 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:16,798 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:17,668 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:17,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:18,799 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:20,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:20,882 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:20,883 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:20,888 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:21,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:21,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:23,888 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:25,763 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:25,764 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:25,764 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:25,769 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:25,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:08:25,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:08:25,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:08:26,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:27,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:29,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:34,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:34,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:34,382 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:34,382 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:34,387 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:34,683 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:35,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:36,071 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:08:39,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:40,000 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:40,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:40,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:40,006 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:40,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:40,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:08:40,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:08:40,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:08:41,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:44,543 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:44,548 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:44,548 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:44,553 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:44,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:45,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:45,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:48,719 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:48,719 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:48,720 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:48,724 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:49,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:49,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:50,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:52,711 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:52,712 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:52,712 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:52,717 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:53,705 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:53,705 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:08:55,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:08:55,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:08:55,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:08:56,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:56,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:56,275 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:08:56,275 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:56,280 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:56,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:08:57,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:01,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:02,070 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:02,071 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:02,071 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:02,076 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:02,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:03,715 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:05,370 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:05,371 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:05,371 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:05,377 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:05,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:06,073 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:09:07,074 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:07,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:10,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:10,894 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:10,895 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:10,900 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:10,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:09:10,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:09:10,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:09:11,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:11,726 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:12,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:16,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:16,564 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:16,564 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:16,570 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:16,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:17,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:17,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:22,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:22,686 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:22,686 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:22,687 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:22,691 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:22,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:23,737 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:25,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:09:25,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:09:25,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:09:27,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:27,553 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:27,553 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:27,558 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:27,740 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:28,559 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:29,743 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:31,199 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:31,199 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:31,200 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:31,205 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:31,744 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:33,747 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:34,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:35,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:35,331 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:35,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:35,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:35,748 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:36,075 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:09:37,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:40,077 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:40,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:40,887 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:40,887 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:40,892 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:40,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:09:40,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:09:40,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:09:41,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:41,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:45,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:45,659 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:45,660 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:45,661 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:45,665 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:45,758 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:47,763 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:50,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:50,225 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:50,225 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:50,230 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:50,765 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:51,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:51,767 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:09:55,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:09:55,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:09:55,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:09:57,216 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:58,630 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:58,631 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:09:58,631 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:58,636 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:58,772 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:09:59,774 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:02,636 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:06,081 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:10:07,649 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:07,650 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:10:07,650 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:07,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:07,655 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:07,780 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:10:09,783 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:10,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:10:10,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:10:10,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:10:13,131 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:13,132 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:10:13,132 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:13,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:13,137 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:13,786 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:10:15,789 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:17,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:17,163 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:10:17,163 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:17,169 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:17,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:10:18,170 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:19,793 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:20,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:20,751 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:10:20,751 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:20,757 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:20,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:10:21,796 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:23,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:24,269 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:24,269 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:10:24,270 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:24,275 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:24,798 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:10:25,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:25,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:10:25,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:10:25,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:10:28,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:28,013 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:10:28,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:28,018 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:28,803 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:10:29,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:29,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:34,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:36,079 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:10:36,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:36,439 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:10:36,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:36,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:36,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:10:37,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:39,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:40,116 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:40,116 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:10:40,117 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:40,121 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:40,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:10:40,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:10:40,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:10:40,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:10:41,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:45,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:48,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:48,277 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:10:48,277 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:48,282 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:48,821 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:10:49,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:50,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:52,087 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:52,088 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:10:52,088 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:52,093 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:52,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:10:53,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:10:55,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:10:55,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:10:55,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:10:56,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:00,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:00,482 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:00,482 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:00,487 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:00,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:01,487 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:01,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:04,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:04,285 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:04,286 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:04,291 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:04,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:05,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:06,084 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:11:07,086 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:08,118 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:08,119 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:08,119 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:08,125 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:08,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:09,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:10,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:11:10,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:11:10,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:11:12,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:12,412 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:12,413 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:12,414 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:12,418 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:12,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:13,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:16,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:16,689 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:16,690 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:16,695 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:16,851 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:17,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:17,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:20,686 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:20,687 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:20,687 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:20,692 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:20,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:21,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:23,693 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:25,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:11:25,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:11:25,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:11:28,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:28,923 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:28,923 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:28,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:28,928 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:29,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:29,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:33,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:34,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:34,576 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:34,576 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:34,581 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:34,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:35,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:36,084 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:11:38,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:38,243 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:38,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:38,248 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:38,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:39,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:39,874 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:40,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:11:40,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:11:40,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:11:42,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:42,742 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:42,742 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:42,747 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:42,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:43,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:44,747 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:46,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:46,474 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:46,475 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:46,480 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:46,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:47,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:50,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:50,202 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:50,203 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:50,203 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:50,208 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:50,885 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:51,888 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:54,118 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:54,119 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:54,119 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:54,124 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:54,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:55,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:11:55,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:11:55,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:11:55,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:11:56,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:57,692 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:57,693 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:11:57,693 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:57,699 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:57,893 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:11:59,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:01,700 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:04,608 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:04,609 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:04,609 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:04,614 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:04,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:05,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:06,088 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:12:07,090 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:09,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:09,023 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:09,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:09,028 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:09,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:09,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:10,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:12:10,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:12:10,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:12:12,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:13,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:13,154 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:13,154 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:13,159 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:13,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:15,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:16,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:16,895 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:16,895 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:16,900 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:16,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:17,900 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:17,916 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:20,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:20,605 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:20,606 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:20,611 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:20,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:21,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:23,612 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:24,505 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:24,506 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:24,506 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:24,511 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:24,922 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:25,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:25,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:12:25,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:12:25,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:12:29,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:29,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:29,293 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:29,293 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:29,298 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:29,927 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:31,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:34,072 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:34,072 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:34,073 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:34,078 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:34,932 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:35,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:35,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:36,088 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:12:38,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:38,634 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:38,634 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:38,639 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:38,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:39,939 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:40,640 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:40,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:12:40,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:12:40,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:12:43,459 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:43,460 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:43,460 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:43,466 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:43,942 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:45,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:46,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:47,624 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:47,625 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:47,625 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:47,630 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:47,948 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:49,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:51,523 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:51,524 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:51,524 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:51,524 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:51,529 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:51,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:12:53,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:12:55,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:12:55,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:12:55,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:12:57,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:59,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:59,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:12:59,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:59,996 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:00,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:01,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:02,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:06,090 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:13:07,478 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:07,479 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:07,479 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:07,485 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:07,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:08,486 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:09,968 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:10,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:13:10,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:13:10,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:13:11,843 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:11,843 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:11,844 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:11,849 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:11,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:13,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:13,973 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:15,651 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:15,652 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:15,652 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:15,657 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:15,974 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:17,977 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:19,658 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:20,875 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:20,876 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:20,876 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:20,881 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:20,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:21,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:24,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:25,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:25,592 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:25,592 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:25,597 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:25,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:13:25,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:13:25,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:13:25,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:27,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:29,749 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:29,749 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:29,750 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:29,755 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:29,988 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:30,755 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:31,991 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:33,429 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:33,430 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:33,430 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:33,435 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:33,993 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:35,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:36,092 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:13:36,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:37,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:37,294 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:37,294 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:37,299 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:37,997 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:40,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:40,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:13:40,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:13:40,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:13:41,182 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:41,899 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:41,900 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:41,900 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:41,905 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:42,002 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:44,005 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:45,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:45,724 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:45,725 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:45,730 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:46,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:46,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:48,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:49,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:49,806 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:49,807 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:49,812 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:50,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:51,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:52,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:54,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:54,464 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:54,465 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:54,470 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:55,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:13:55,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:13:55,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:13:55,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:13:56,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:13:57,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:59,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:59,059 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:13:59,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:59,065 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:00,023 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:14:00,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:14:03,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:05,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:05,958 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:14:05,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:05,963 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:06,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:14:06,094 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:14:08,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:14:09,095 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:10,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:14:10,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:14:10,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:14:11,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:11,849 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:14:11,849 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:11,854 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:12,034 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:14:14,038 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:14:14,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:19,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:20,362 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:20,363 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:14:20,363 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:20,368 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:21,042 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:14:22,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:14:25,369 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:25,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:14:25,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:14:25,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:14:28,900 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:28,900 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:14:28,901 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:28,906 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:29,047 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:14:30,049 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:14:30,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:34,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:34,390 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:14:34,390 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:34,395 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:35,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:14:36,055 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:14:36,099 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:14:36,100 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:39,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:39,298 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:14:39,299 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:39,303 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:40,059 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:14:40,059 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:14:40,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:14:40,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:14:40,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:14:41,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:43,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:43,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:14:43,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:43,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:44,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:14:46,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:14:47,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:48,299 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:48,300 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:14:48,300 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:48,305 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:49,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:14:50,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:14:52,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:53,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:53,918 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:14:53,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:53,924 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:54,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:14:55,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:14:55,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:14:55,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:14:56,075 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:14:58,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:58,333 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:58,333 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:14:58,333 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:58,339 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:59,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:00,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:03,378 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:05,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:05,442 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:05,442 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:05,447 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:06,084 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:06,098 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:15:08,087 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:09,100 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:09,531 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:09,531 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:09,531 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:09,536 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:10,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:10,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:15:10,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:15:10,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:15:12,092 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:14,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:14,752 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:14,753 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:14,753 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:14,758 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:15,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:16,097 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:19,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:19,607 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:19,608 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:19,608 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:19,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:20,099 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:22,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:23,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:23,592 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:23,592 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:23,597 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:24,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:25,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:25,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:15:25,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:15:25,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:15:26,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:29,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:29,567 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:29,567 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:29,572 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:30,109 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:30,610 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:32,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:34,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:34,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:34,258 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:34,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:35,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:36,103 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:15:36,104 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:36,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:40,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:15:40,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:15:40,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:15:41,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:42,728 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:42,729 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:42,729 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:42,734 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:43,120 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:44,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:46,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:46,751 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:46,751 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:46,751 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:46,756 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:47,124 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:48,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:50,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:50,991 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:50,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:50,996 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:51,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:51,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:52,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:55,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:55,213 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:55,213 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:55,218 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:55,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:15:55,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:15:55,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:15:56,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:15:56,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:15:57,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:59,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:59,606 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:15:59,606 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:59,611 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:00,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:02,143 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:02,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:06,103 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:16:06,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:06,504 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:06,504 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:06,509 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:07,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:08,149 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:08,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:10,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:16:10,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:16:10,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:16:12,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:12,027 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:12,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:12,032 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:12,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:14,070 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:14,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:18,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:18,143 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:18,144 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:18,148 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:18,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:19,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:20,161 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:22,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:22,453 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:22,454 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:22,459 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:23,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:24,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:24,459 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:25,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:16:25,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:16:25,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:16:26,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:26,315 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:26,315 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:26,319 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:27,167 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:28,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:30,228 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:30,229 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:30,229 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:30,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:30,234 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:31,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:32,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:35,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:36,108 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:16:39,379 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:39,380 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:39,380 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:39,385 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:40,181 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:40,181 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:40,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:40,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:16:40,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:16:40,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:16:43,187 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:43,187 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:43,188 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:43,192 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:44,185 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:44,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:46,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:47,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:47,346 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:47,347 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:47,351 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:48,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:48,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:51,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:51,338 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:51,338 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:51,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:51,343 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:52,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:52,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:16:55,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:16:55,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:16:55,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:16:56,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:56,809 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:16:56,810 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:56,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:56,815 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:57,198 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:16:58,200 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:00,483 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:00,484 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:00,484 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:00,490 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:01,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:02,205 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:02,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:04,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:04,790 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:04,791 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:04,796 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:05,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:06,107 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:17:06,210 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:08,108 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:08,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:08,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:08,854 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:08,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:09,212 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:10,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:10,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:17:10,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:17:10,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:17:13,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:14,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:14,534 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:14,535 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:14,539 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:15,218 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:16,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:18,578 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:19,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:19,993 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:19,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:19,999 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:20,223 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:22,226 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:23,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:24,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:24,451 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:24,451 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:24,456 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:25,228 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:25,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:17:25,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:17:25,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:17:26,228 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:29,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:29,586 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:29,587 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:29,587 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:29,592 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:30,231 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:32,234 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:33,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:33,569 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:33,570 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:33,575 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:34,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:34,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:36,109 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:17:36,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:37,435 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:37,435 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:37,436 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:37,441 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:38,241 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:38,242 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:40,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:40,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:17:40,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:17:40,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:17:42,124 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:42,125 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:42,125 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:42,130 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:42,244 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:44,247 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:45,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:45,940 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:45,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:45,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:45,945 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:46,248 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:48,252 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:50,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:51,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:51,562 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:51,562 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:51,568 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:52,254 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:54,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:17:55,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:17:55,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:55,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:17:55,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:17:56,661 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:56,661 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:17:56,662 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:56,667 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:57,259 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:17:58,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:00,692 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:00,692 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:00,693 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:00,698 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:01,263 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:01,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:02,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:04,528 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:04,529 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:04,529 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:04,534 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:05,266 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:06,111 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:18:06,270 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:07,112 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:08,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:08,464 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:08,464 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:08,469 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:09,271 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:10,274 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:10,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:18:10,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:18:10,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:18:12,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:12,275 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:12,275 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:12,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:12,280 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:13,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:14,279 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:16,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:16,018 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:16,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:16,024 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:16,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:18,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:18,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:22,513 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:22,514 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:22,514 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:22,519 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:23,287 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:23,519 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:24,290 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:25,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:18:25,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:18:25,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:18:27,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:27,374 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:27,375 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:27,380 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:28,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:28,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:29,418 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:34,418 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:35,710 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:35,710 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:35,711 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:35,716 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:36,113 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:18:36,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:38,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:39,714 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:39,715 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:39,715 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:39,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:39,720 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:40,303 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:40,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:18:40,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:18:40,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:18:42,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:43,679 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:43,680 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:43,680 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:43,685 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:44,307 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:44,722 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:46,310 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:47,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:47,262 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:47,263 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:47,268 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:47,311 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:48,315 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:50,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:52,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:52,917 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:52,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:52,922 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:53,318 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:18:54,321 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:18:55,923 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:55,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:18:55,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:18:55,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:18:58,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:58,510 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:18:58,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:58,516 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:59,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:00,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:01,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:02,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:02,223 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:02,224 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:02,228 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:02,328 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:04,331 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:06,118 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:19:07,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:07,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:07,617 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:07,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:07,622 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:08,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:08,337 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:10,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:19:10,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:19:10,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:19:11,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:11,975 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:11,975 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:11,980 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:12,339 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:13,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:14,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:15,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:15,903 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:15,903 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:15,908 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:16,344 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:18,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:18,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:23,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:23,084 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:23,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:23,089 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:23,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:24,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:24,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:25,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:19:25,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:19:25,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:19:27,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:27,084 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:27,085 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:27,090 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:27,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:28,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:29,126 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:30,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:30,622 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:30,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:30,627 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:31,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:32,361 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:34,628 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:34,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:34,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:34,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:34,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:35,363 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:36,117 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:19:36,365 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:39,354 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:39,355 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:39,355 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:39,361 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:39,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:40,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:40,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:40,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:19:40,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:19:40,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:19:45,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:45,549 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:45,549 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:45,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:45,554 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:46,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:46,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:49,539 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:49,540 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:49,540 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:49,545 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:50,380 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:50,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:51,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:53,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:53,485 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:53,485 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:53,490 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:54,385 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:54,385 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:19:55,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:19:55,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:19:55,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:19:57,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:57,032 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:19:57,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:57,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:57,037 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:57,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:19:58,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:02,074 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:05,390 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:05,391 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:05,391 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:05,396 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:05,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:06,122 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:20:06,400 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:07,124 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:09,282 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:09,283 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:09,283 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:09,288 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:09,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:10,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:10,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:20:10,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:20:10,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:20:12,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:12,775 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:12,776 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:12,776 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:12,781 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:13,406 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:14,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:17,080 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:17,081 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:17,081 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:17,086 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:17,411 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:18,086 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:18,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:22,435 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:22,436 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:22,436 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:22,441 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:23,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:23,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:24,420 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:25,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:20:25,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:20:25,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:20:29,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:30,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:30,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:30,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:30,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:31,423 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:32,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:35,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:35,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:35,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:35,326 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:35,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:35,428 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:36,121 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:20:36,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:39,107 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:39,108 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:39,108 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:39,113 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:39,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:40,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:40,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:40,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:20:40,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:20:40,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:20:42,871 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:42,872 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:42,872 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:42,877 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:43,436 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:44,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:45,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:47,324 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:47,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:47,325 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:47,330 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:47,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:48,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:51,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:51,371 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:51,372 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:51,372 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:51,377 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:51,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:52,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:55,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:55,315 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:55,315 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:55,320 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:55,450 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:20:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:20:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:20:55,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:20:56,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:20:57,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:59,101 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:59,102 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:20:59,102 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:59,107 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:59,452 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:00,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:03,108 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:03,912 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:03,913 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:03,913 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:03,918 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:04,457 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:06,126 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:21:06,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:08,127 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:10,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:21:10,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:21:10,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:21:12,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:12,477 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:12,477 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:12,482 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:13,465 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:13,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:14,468 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:17,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:17,059 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:17,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:17,065 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:17,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:18,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:19,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:20,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:20,881 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:20,881 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:20,886 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:21,475 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:22,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:24,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:25,098 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:25,099 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:25,099 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:25,104 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:25,480 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:25,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:21:25,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:21:25,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:21:26,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:30,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:33,638 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:33,638 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:33,639 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:33,643 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:34,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:34,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:35,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:36,125 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:21:38,851 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:38,852 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:38,852 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:38,857 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:39,491 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:40,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:40,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:40,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:21:40,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:21:40,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:21:44,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:44,490 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:44,490 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:44,495 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:44,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:46,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:46,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:48,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:48,407 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:48,407 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:48,412 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:48,501 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:50,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:51,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:51,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:51,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:51,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:51,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:52,505 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:54,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:21:55,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:21:55,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:21:55,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:21:56,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:56,289 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:21:56,289 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:56,295 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:56,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:21:57,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:58,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:00,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:00,015 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:00,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:00,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:00,514 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:02,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:03,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:05,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:05,483 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:05,483 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:05,488 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:05,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:06,128 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:22:06,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:08,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:10,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:22:10,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:22:10,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:22:11,155 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:11,253 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:11,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:11,258 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:11,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:12,528 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:13,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:18,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:19,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:19,469 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:19,469 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:19,474 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:19,532 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:20,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:23,131 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:23,131 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:23,132 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:23,137 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:23,537 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:24,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:24,540 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:25,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:22:25,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:22:25,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:22:28,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:28,985 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:28,986 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:28,991 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:29,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:30,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:30,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:32,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:32,823 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:32,823 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:32,828 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:33,547 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:34,549 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:35,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:36,130 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:22:40,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:22:40,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:22:40,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:22:40,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:41,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:41,321 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:41,321 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:41,326 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:41,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:42,556 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:46,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:47,542 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:47,543 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:47,543 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:47,548 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:47,559 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:48,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:50,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:50,821 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:50,821 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:50,827 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:51,564 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:51,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:52,567 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:22:55,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:22:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:22:55,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:22:56,216 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:56,216 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:22:56,216 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:56,221 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:56,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:22:57,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:58,572 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:02,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:04,770 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:04,771 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:04,771 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:04,776 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:05,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:06,132 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:23:06,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:08,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:08,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:08,550 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:08,550 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:08,555 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:08,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:10,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:10,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:23:10,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:23:10,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:23:12,539 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:12,540 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:12,540 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:12,544 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:12,586 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:13,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:14,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:15,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:15,969 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:15,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:15,974 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:16,590 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:18,593 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:18,975 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:20,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:20,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:20,462 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:20,467 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:20,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:22,598 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:24,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:25,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:23:25,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:23:25,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:23:26,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:26,290 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:26,290 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:26,295 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:26,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:28,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:29,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:29,836 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:29,836 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:29,837 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:29,841 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:30,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:30,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:34,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:34,008 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:34,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:34,014 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:34,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:35,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:36,134 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:23:36,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:40,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:40,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:23:40,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:23:40,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:23:43,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:43,392 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:43,393 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:43,397 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:43,616 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:44,619 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:45,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:50,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:51,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:51,744 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:51,745 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:51,750 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:52,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:52,627 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:55,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:55,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:55,258 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:55,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:55,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:23:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:23:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:23:55,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:23:56,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:23:59,183 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:59,183 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:23:59,183 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:59,188 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:59,632 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:00,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:01,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:03,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:03,751 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:03,751 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:03,757 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:04,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:04,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:06,136 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:24:07,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:07,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:07,550 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:07,550 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:07,556 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:07,643 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:08,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:10,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:24:10,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:24:10,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:24:12,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:13,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:13,295 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:13,295 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:13,300 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:13,649 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:14,651 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:16,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:16,561 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:16,562 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:16,567 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:16,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:17,567 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:18,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:20,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:20,154 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:20,154 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:20,159 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:20,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:22,660 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:23,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:25,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:24:25,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:24:25,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:24:28,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:28,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:28,690 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:28,690 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:28,695 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:29,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:30,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:33,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:33,275 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:33,276 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:33,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:33,281 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:33,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:34,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:36,141 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:24:39,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:39,766 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:39,767 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:39,767 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:39,772 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:40,677 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:40,677 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:40,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:24:40,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:24:40,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:24:43,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:43,989 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:43,989 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:43,994 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:44,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:45,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:46,682 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:48,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:48,516 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:48,516 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:48,521 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:48,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:50,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:50,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:53,674 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:53,675 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:53,675 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:53,681 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:53,689 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:24:54,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:24:55,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:24:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:24:55,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:24:59,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:59,581 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:24:59,581 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:59,586 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:59,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:00,697 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:01,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:04,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:04,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:04,990 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:04,995 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:05,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:06,140 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:25:06,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:07,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:10,594 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:10,595 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:10,595 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:10,600 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:10,706 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:10,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:25:10,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:25:10,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:25:12,298 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:12,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:17,298 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:17,416 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:17,417 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:17,417 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:17,422 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:17,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:18,715 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:22,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:22,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:22,278 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:22,283 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:22,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:23,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:24,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:25,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:25:25,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:25:25,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:25:26,092 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:26,306 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:26,306 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:26,311 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:26,722 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:28,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:28,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:29,786 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:29,786 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:29,787 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:29,792 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:30,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:30,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:33,792 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:33,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:33,938 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:33,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:33,944 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:34,732 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:34,732 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:36,145 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:25:39,146 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:39,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:39,678 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:39,678 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:39,683 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:39,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:40,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:40,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:25:40,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:25:40,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:25:44,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:44,475 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:44,476 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:44,476 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:44,481 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:44,740 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:46,743 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:49,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:49,155 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:49,155 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:49,160 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:49,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:50,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:50,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:53,775 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:53,776 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:53,777 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:53,781 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:54,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:25:54,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:25:55,782 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:25:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:25:55,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:25:58,635 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:58,636 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:25:58,637 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:58,642 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:58,755 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:00,758 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:01,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:04,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:04,699 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:04,699 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:04,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:04,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:06,144 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:26:06,766 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:07,145 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:09,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:09,923 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:09,923 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:09,928 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:10,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:10,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:10,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:26:10,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:26:10,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:26:12,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:13,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:13,860 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:13,860 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:13,865 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:14,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:14,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:17,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:19,918 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:19,919 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:19,920 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:19,925 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:20,780 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:20,780 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:22,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:23,533 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:23,534 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:23,534 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:23,540 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:23,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:24,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:25,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:26:25,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:26:25,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:26:27,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:27,117 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:27,118 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:27,123 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:27,786 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:28,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:28,789 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:32,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:32,739 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:32,739 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:32,744 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:32,791 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:33,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:34,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:36,146 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:26:37,713 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:37,714 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:37,714 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:37,719 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:37,796 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:38,798 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:39,720 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:40,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:26:40,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:26:40,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:26:41,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:41,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:41,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:41,516 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:41,800 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:42,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:45,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:48,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:48,897 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:48,897 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:48,902 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:49,807 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:50,810 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:50,903 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:53,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:53,286 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:53,287 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:53,291 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:53,812 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:54,814 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:26:55,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:26:55,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:26:55,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:26:55,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:57,070 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:57,071 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:26:57,071 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:57,076 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:57,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:26:58,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:00,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:00,809 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:00,809 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:00,814 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:00,820 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:01,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:02,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:04,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:04,483 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:04,483 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:04,488 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:04,824 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:06,149 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:27:06,829 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:07,150 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:08,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:08,407 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:08,407 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:08,412 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:08,830 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:10,833 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:10,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:27:10,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:27:10,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:27:12,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:13,816 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:13,817 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:13,817 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:13,822 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:13,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:14,838 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:17,743 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:17,744 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:17,744 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:17,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:17,749 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:17,839 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:18,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:22,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:23,756 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:23,757 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:23,758 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:23,762 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:23,845 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:24,848 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:25,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:27:25,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:27:25,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:27:28,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:29,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:29,051 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:29,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:29,056 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:29,851 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:30,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:34,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:35,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:35,374 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:35,374 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:35,379 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:35,856 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:36,151 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:27:36,859 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:39,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:39,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:39,404 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:39,404 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:39,409 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:39,860 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:40,863 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:40,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:27:40,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:27:40,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:27:43,754 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:43,755 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:43,755 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:43,760 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:43,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:44,798 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:46,868 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:47,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:47,585 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:47,586 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:47,591 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:47,869 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:48,872 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:50,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:53,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:53,261 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:53,261 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:53,266 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:53,875 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:54,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:27:55,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:27:55,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:27:55,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:27:55,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:57,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:57,549 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:27:57,550 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:57,555 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:57,879 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:27:58,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:01,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:01,331 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:01,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:01,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:01,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:01,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:02,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:05,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:05,648 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:05,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:05,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:05,889 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:06,153 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:28:06,893 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:07,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:10,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:28:10,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:28:10,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:28:12,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:12,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:12,327 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:12,327 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:12,332 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:12,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:14,900 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:17,370 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:18,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:18,120 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:18,120 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:18,125 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:18,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:20,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:21,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:21,993 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:21,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:21,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:22,907 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:22,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:24,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:25,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:25,207 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:25,207 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:25,212 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:25,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:25,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:28:25,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:28:25,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:28:26,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:28,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:33,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:33,485 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:33,486 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:33,486 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:33,491 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:33,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:34,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:36,155 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:28:39,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:40,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:28:40,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:28:40,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:28:41,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:41,625 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:41,626 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:41,631 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:41,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:42,927 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:44,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:46,494 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:46,495 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:46,495 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:46,500 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:46,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:48,934 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:50,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:51,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:51,561 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:51,561 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:51,566 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:51,936 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:52,939 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:28:55,567 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:55,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:28:55,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:28:55,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:28:57,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:57,243 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:28:57,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:57,248 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:57,942 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:28:58,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:01,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:01,388 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:01,389 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:01,389 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:01,394 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:01,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:02,949 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:05,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:05,739 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:05,739 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:05,744 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:05,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:06,160 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:29:06,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:07,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:09,919 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:09,920 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:09,920 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:09,926 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:09,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:10,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:29:10,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:29:10,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:29:12,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:12,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:13,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:13,727 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:13,727 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:13,732 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:13,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:14,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:17,770 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:18,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:18,809 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:18,809 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:18,814 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:18,965 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:20,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:22,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:22,727 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:22,728 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:22,733 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:22,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:23,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:24,973 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:25,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:29:25,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:29:25,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:29:27,525 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:27,526 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:27,526 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:27,531 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:27,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:28,978 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:29,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:32,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:32,016 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:32,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:32,022 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:32,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:34,983 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:35,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:36,159 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:29:37,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:37,356 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:37,356 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:37,361 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:37,985 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:38,988 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:40,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:40,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:29:40,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:29:40,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:29:46,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:46,550 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:46,551 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:46,551 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:46,556 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:46,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:48,994 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:49,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:49,887 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:49,887 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:49,893 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:49,995 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:51,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:52,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:53,895 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:53,896 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:53,896 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:53,901 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:53,999 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:29:55,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:29:55,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:29:55,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:29:57,002 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:29:57,228 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:59,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:59,248 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:29:59,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:59,254 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:00,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:30:01,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:30:02,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:06,164 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:30:07,656 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:07,657 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:30:07,657 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:07,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:07,662 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:08,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:30:09,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:30:10,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:30:10,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:30:10,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:30:13,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:18,184 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:18,185 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:30:18,185 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:18,190 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:19,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:30:19,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:21,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:30:21,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:21,936 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:30:21,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:21,941 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:22,025 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:30:24,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:25,026 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:30:25,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:30:25,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:30:25,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:30:26,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:26,807 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:30:26,807 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:26,812 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:27,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:30:29,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:30:30,850 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:30,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:30,955 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:30:30,956 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:30,960 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:31,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:30:33,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:30:35,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:36,164 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:30:40,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:40,143 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:30:40,143 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:40,148 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:40,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:30:40,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:40,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:30:40,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:30:41,040 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:30:43,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:30:46,413 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:47,181 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:47,181 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:30:47,182 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:47,186 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:48,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:30:49,049 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:30:52,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:54,372 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:54,373 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:30:54,373 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:54,378 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:55,053 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:30:55,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:30:55,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:30:55,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:30:57,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:30:58,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:58,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:58,846 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:30:58,846 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:58,851 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:59,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:01,060 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:03,754 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:03,755 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:03,755 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:03,756 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:03,760 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:04,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:05,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:06,165 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:31:07,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:07,244 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:07,244 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:07,249 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:08,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:09,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:09,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:10,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:31:10,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:31:10,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:31:11,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:11,249 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:11,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:11,254 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:12,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:13,075 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:14,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:15,878 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:15,879 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:15,879 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:15,884 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:16,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:17,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:19,884 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:20,422 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:20,423 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:20,423 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:20,428 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:21,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:23,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:25,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:25,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:31:25,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:31:25,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:31:28,585 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:28,586 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:28,586 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:28,591 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:29,088 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:30,630 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:31,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:32,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:32,428 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:32,428 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:32,433 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:33,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:35,093 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:36,174 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:31:36,175 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:38,327 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:38,328 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:38,328 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:38,333 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:39,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:40,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:31:40,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:31:40,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:31:41,099 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:41,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:43,789 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:43,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:43,790 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:43,795 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:44,101 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:45,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:46,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:48,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:48,510 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:48,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:48,515 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:49,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:51,110 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:52,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:53,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:53,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:53,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:53,997 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:54,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:31:55,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:31:55,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:31:55,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:31:57,115 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:31:58,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:59,168 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:59,169 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:31:59,169 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:59,174 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:00,117 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:01,120 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:03,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:03,968 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:03,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:03,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:03,974 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:04,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:05,124 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:06,170 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:32:08,462 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:08,463 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:08,463 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:08,469 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:09,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:09,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:10,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:32:10,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:32:10,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:32:11,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:12,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:12,778 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:12,778 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:12,783 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:13,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:14,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:15,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:16,525 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:16,526 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:16,526 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:16,531 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:17,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:19,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:20,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:20,815 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:20,816 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:20,816 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:20,821 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:21,141 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:23,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:24,782 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:24,783 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:24,783 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:24,788 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:25,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:25,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:25,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:32:25,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:32:25,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:32:27,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:28,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:28,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:28,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:28,937 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:29,149 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:30,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:31,153 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:34,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:34,290 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:34,290 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:34,295 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:35,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:36,175 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:32:36,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:37,158 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:38,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:38,592 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:38,592 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:38,597 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:39,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:40,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:32:40,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:32:40,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:32:41,162 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:41,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:43,631 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:43,631 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:43,632 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:43,637 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:44,164 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:45,166 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:46,674 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:47,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:47,839 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:47,839 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:47,845 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:48,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:49,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:51,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:52,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:52,011 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:52,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:52,017 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:52,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:53,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:32:55,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:32:55,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:32:55,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:32:57,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:57,977 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:57,977 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:32:57,978 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:57,982 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:58,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:32:59,182 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:01,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:01,887 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:01,888 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:01,892 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:02,183 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:02,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:03,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:05,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:05,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:05,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:05,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:06,174 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:33:06,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:07,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:08,175 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:10,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:10,651 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:10,652 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:10,656 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:10,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:33:10,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:33:10,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:33:11,194 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:13,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:13,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:14,310 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:14,310 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:14,311 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:14,316 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:15,198 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:17,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:18,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:18,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:18,485 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:18,485 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:18,490 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:19,203 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:21,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:22,122 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:22,123 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:22,123 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:22,128 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:22,207 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:24,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:25,210 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:25,816 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:25,817 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:25,817 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:25,822 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:25,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:33:25,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:33:25,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:33:26,211 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:27,213 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:29,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:29,420 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:29,421 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:29,421 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:29,426 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:30,215 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:31,218 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:34,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:34,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:34,670 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:34,670 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:34,675 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:35,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:36,179 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:33:37,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:38,115 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:38,116 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:38,116 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:38,121 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:38,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:40,121 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:40,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:33:40,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:33:40,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:33:41,228 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:41,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:41,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:41,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:41,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:42,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:43,231 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:45,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:48,583 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:48,584 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:48,584 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:48,589 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:49,236 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:51,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:51,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:52,184 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:52,185 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:33:52,185 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:52,190 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:52,240 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:33:55,244 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:33:55,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:33:55,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:33:55,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:33:57,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:00,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:00,546 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:00,546 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:00,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:01,247 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:02,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:03,250 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:03,976 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:03,977 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:03,977 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:03,982 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:04,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:05,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:06,178 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:34:08,179 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:08,345 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:08,346 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:08,346 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:08,351 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:09,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:10,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:34:10,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:34:10,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:34:11,259 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:12,349 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:12,350 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:12,350 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:12,356 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:13,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:13,393 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:15,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:16,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:16,542 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:16,542 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:16,549 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:17,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:18,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:19,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:20,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:20,644 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:20,645 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:20,649 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:21,270 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:23,274 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:23,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:24,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:24,267 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:24,267 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:24,272 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:24,274 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:25,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:34:25,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:34:25,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:34:27,278 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:27,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:27,874 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:27,874 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:27,879 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:28,278 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:28,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:29,281 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:33,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:33,195 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:33,195 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:33,201 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:33,283 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:34,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:35,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:36,183 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:34:37,184 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:37,184 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:37,185 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:37,189 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:37,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:39,291 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:40,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:40,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:34:40,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:34:40,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:34:41,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:41,556 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:41,556 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:41,561 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:42,292 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:43,295 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:45,370 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:45,371 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:45,371 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:45,371 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:45,376 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:46,297 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:47,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:49,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:49,415 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:49,415 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:49,420 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:50,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:50,420 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:51,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:54,378 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:54,378 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:54,379 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:54,384 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:55,307 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:34:55,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:34:55,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:55,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:34:55,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:34:57,310 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:34:58,986 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:58,986 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:34:58,987 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:58,991 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:59,311 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:01,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:01,313 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:02,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:02,963 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:02,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:02,968 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:03,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:05,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:06,182 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:35:06,184 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:06,735 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:06,736 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:06,736 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:06,742 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:07,320 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:09,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:10,305 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:10,306 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:10,306 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:10,311 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:10,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:10,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:35:10,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:35:10,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:35:11,210 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:13,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:16,210 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:17,489 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:17,489 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:17,489 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:17,495 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:18,331 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:19,333 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:20,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:20,923 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:20,923 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:20,928 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:21,335 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:21,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:23,338 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:25,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:25,553 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:25,553 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:25,558 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:25,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:35:25,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:35:25,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:35:26,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:27,318 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:27,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:29,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:29,257 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:29,257 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:29,262 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:29,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:31,346 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:33,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:36,184 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:35:36,740 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:36,744 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:36,744 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:36,749 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:37,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:38,749 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:39,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:40,536 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:40,537 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:40,537 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:40,542 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:40,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:35:40,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:35:40,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:35:41,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:43,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:44,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:44,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:44,684 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:44,684 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:44,689 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:45,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:47,361 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:49,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:49,097 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:49,097 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:49,102 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:49,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:50,103 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:51,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:53,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:53,577 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:53,577 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:53,582 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:54,369 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:55,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:35:55,583 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:55,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:35:55,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:35:55,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:35:57,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:57,286 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:35:57,286 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:57,291 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:57,372 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:35:59,376 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:01,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:01,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:01,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:01,438 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:01,443 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:02,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:03,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:06,187 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:36:06,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:06,254 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:06,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:06,259 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:06,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:07,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:07,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:10,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:10,047 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:10,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:10,052 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:10,388 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:10,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:36:10,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:36:10,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:36:11,391 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:13,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:13,311 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:13,311 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:13,312 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:13,316 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:13,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:15,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:18,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:20,495 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:20,495 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:20,496 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:20,500 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:21,399 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:23,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:23,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:25,708 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:25,709 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:25,709 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:25,714 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:25,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:36:25,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:36:25,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:36:26,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:27,407 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:29,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:30,883 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:30,884 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:30,884 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:30,890 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:31,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:33,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:34,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:35,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:35,063 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:35,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:35,068 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:35,413 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:36,189 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:36:37,416 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:39,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:39,949 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:39,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:39,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:39,954 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:40,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:40,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:36:40,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:36:40,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:36:41,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:45,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:45,900 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:45,901 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:45,901 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:45,906 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:46,424 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:47,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:49,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:49,876 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:49,876 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:49,881 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:50,429 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:50,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:51,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:36:55,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:55,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:36:55,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:36:55,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:36:56,435 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:56,435 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:36:56,435 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:56,440 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:57,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:36:59,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:01,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:01,336 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:01,336 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:01,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:01,341 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:01,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:03,442 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:06,191 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:37:07,192 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:07,349 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:07,350 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:07,350 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:07,355 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:07,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:09,449 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:10,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:37:10,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:37:10,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:37:11,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:11,383 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:11,384 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:11,388 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:11,450 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:12,425 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:13,454 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:17,425 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:17,485 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:17,486 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:17,486 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:17,491 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:18,457 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:19,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:21,299 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:21,300 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:21,300 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:21,306 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:21,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:23,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:23,469 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:25,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:25,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:25,278 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:25,283 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:25,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:25,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:37:25,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:37:25,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:37:27,475 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:28,310 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:29,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:29,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:29,438 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:29,443 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:29,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:31,480 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:33,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:34,533 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:34,534 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:34,534 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:34,539 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:35,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:36,193 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:37:37,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:39,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:39,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:39,239 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:39,239 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:39,244 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:39,490 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:40,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:37:40,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:37:40,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:37:41,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:43,472 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:43,473 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:43,473 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:43,478 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:43,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:44,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:45,500 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:47,879 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:47,880 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:47,880 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:47,885 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:48,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:49,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:49,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:51,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:51,880 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:51,881 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:51,886 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:52,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:53,512 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:37:54,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:55,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:37:55,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:37:55,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:37:56,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:56,989 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:37:56,989 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:56,994 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:57,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:37:59,520 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:00,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:00,721 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:00,722 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:00,722 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:00,727 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:01,522 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:03,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:04,908 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:04,908 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:04,909 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:04,913 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:05,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:05,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:06,198 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:38:07,533 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:08,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:08,505 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:08,505 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:08,510 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:08,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:10,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:38:10,999 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:38:10,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:38:11,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:11,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:13,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:13,138 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:13,138 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:13,143 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:13,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:15,544 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:16,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:16,942 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:16,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:16,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:16,947 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:17,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:19,549 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:20,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:20,501 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:20,502 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:20,507 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:20,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:22,507 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:23,556 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:24,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:24,705 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:24,705 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:24,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:25,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:25,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:38:25,999 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:38:25,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:38:27,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:28,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:31,634 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:31,634 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:31,634 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:31,639 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:32,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:33,570 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:33,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:36,197 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:38:36,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:36,202 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:36,202 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:36,207 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:36,571 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:37,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:39,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:40,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:40,601 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:40,601 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:40,606 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:40,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:38:40,999 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:38:41,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:38:41,579 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:43,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:44,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:44,456 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:44,457 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:44,457 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:44,462 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:44,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:47,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:48,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:48,115 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:48,115 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:48,120 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:48,590 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:49,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:50,120 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:52,548 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:52,549 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:52,549 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:52,554 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:52,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:55,555 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:55,601 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:38:56,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:38:56,000 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:38:56,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:38:56,105 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:56,212 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:38:56,213 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:56,218 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:56,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:38:57,604 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:00,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:00,623 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:00,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:00,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:00,628 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:01,608 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:03,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:04,569 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:04,570 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:04,571 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:04,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:04,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:06,202 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:39:06,203 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:07,619 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:09,736 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:09,737 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:09,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:09,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:10,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:11,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:39:11,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:39:11,000 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:39:11,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:11,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:14,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:14,055 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:14,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:14,060 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:14,625 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:15,629 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:17,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:19,699 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:19,700 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:19,700 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:19,705 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:20,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:21,636 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:22,706 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:23,512 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:23,513 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:23,513 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:23,518 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:23,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:25,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:26,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:39:26,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:39:26,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:39:28,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:28,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:28,253 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:28,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:28,258 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:28,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:29,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:33,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:36,201 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:39:36,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:36,833 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:36,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:36,838 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:37,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:38,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:39,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:41,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:39:41,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:39:41,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:39:42,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:42,615 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:42,615 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:42,620 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:42,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:44,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:45,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:47,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:47,177 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:47,177 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:47,182 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:47,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:49,678 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:50,183 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:51,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:51,326 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:51,326 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:51,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:51,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:53,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:39:55,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:56,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:39:56,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:39:56,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:39:56,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:56,261 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:39:56,262 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:56,267 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:56,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:39:57,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:01,305 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:02,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:02,157 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:02,157 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:02,162 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:02,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:03,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:06,206 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:40:07,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:07,130 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:07,130 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:07,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:07,135 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:07,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:09,706 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:11,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:40:11,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:40:11,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:40:12,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:13,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:13,874 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:13,874 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:13,879 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:14,711 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:15,714 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:17,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:19,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:19,267 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:19,267 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:19,272 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:19,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:21,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:23,273 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:23,298 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:23,298 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:23,299 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:23,304 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:23,724 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:25,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:26,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:40:26,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:40:26,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:40:28,706 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:28,707 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:28,707 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:28,707 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:28,712 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:28,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:31,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:33,635 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:33,636 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:33,637 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:33,643 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:33,739 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:34,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:35,743 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:36,205 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:40:37,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:37,563 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:37,563 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:37,568 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:37,745 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:39,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:40,569 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:41,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:40:41,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:40:41,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:40:44,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:44,017 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:44,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:44,023 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:44,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:45,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:46,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:49,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:49,434 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:49,434 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:49,439 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:49,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:51,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:51,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:40:56,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:40:56,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:40:56,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:40:57,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:57,376 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:57,377 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:40:57,377 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:57,382 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:57,772 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:40:59,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:02,417 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:05,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:05,825 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:05,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:05,831 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:06,208 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:41:06,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:07,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:08,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:11,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:41:11,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:41:11,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:41:11,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:11,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:11,438 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:11,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:11,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:13,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:13,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:18,370 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:18,371 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:18,371 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:18,376 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:18,800 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:19,377 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:19,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:21,671 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:21,675 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:21,675 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:21,680 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:21,806 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:23,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:24,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:26,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:41:26,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:41:26,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:41:26,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:26,354 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:26,354 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:26,359 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:26,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:27,817 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:30,397 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:30,528 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:30,528 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:30,529 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:30,533 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:30,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:31,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:34,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:34,562 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:34,563 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:34,568 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:34,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:35,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:35,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:36,210 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:41:39,360 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:39,360 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:39,361 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:39,366 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:39,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:41,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:41:41,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:41:41,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:41:41,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:41,837 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:43,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:43,336 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:43,336 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:43,341 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:43,838 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:45,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:46,377 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:48,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:48,326 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:48,326 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:48,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:48,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:49,848 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:52,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:54,080 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:54,081 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:54,082 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:54,087 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:54,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:55,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:41:56,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:41:56,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:41:56,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:41:58,199 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:58,609 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:58,609 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:41:58,609 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:58,614 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:58,859 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:41:59,863 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:02,362 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:02,363 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:02,363 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:02,368 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:02,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:03,368 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:03,868 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:06,212 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:42:06,489 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:06,490 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:06,490 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:06,495 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:06,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:07,875 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:08,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:10,986 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:10,987 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:10,987 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:10,992 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:11,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:42:11,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:42:11,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:42:11,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:13,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:14,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:15,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:15,206 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:15,206 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:15,211 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:15,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:17,888 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:19,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:19,038 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:19,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:19,043 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:19,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:20,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:21,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:25,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:26,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:42:26,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:42:26,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:42:27,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:27,374 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:27,374 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:27,379 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:27,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:29,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:30,417 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:32,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:32,049 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:32,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:32,054 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:32,907 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:33,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:36,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:36,217 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:42:36,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:36,645 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:36,645 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:36,650 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:36,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:37,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:40,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:40,041 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:40,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:40,047 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:40,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:41,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:42:41,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:42:41,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:42:41,207 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:41,922 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:45,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:45,830 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:45,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:45,835 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:45,926 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:46,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:47,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:49,665 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:49,666 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:49,666 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:49,672 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:49,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:51,938 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:52,672 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:53,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:53,277 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:53,277 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:53,282 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:53,939 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:42:55,943 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:42:56,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:42:56,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:42:56,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:42:58,269 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:58,791 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:58,792 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:42:58,792 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:58,797 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:58,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:01,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:03,378 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:03,378 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:03,379 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:03,379 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:03,384 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:03,953 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:05,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:06,216 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:43:06,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:06,860 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:06,860 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:06,865 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:06,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:08,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:09,963 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:11,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:43:11,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:43:11,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:43:11,172 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:11,292 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:11,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:11,297 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:11,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:13,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:14,298 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:15,087 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:15,088 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:15,088 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:15,093 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:15,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:17,974 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:19,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:19,222 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:19,222 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:19,227 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:19,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:20,228 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:21,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:23,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:23,120 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:23,121 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:23,126 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:23,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:25,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:26,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:43:26,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:43:26,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:43:26,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:29,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:29,617 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:29,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:29,622 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:29,991 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:31,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:31,995 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:33,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:33,409 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:33,409 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:33,414 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:33,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:36,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:36,220 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:43:37,222 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:39,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:39,066 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:39,066 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:39,071 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:40,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:41,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:43:41,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:43:41,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:43:42,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:43,198 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:43,198 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:43,199 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:43,199 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:43,204 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:44,010 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:46,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:46,984 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:46,984 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:46,985 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:46,990 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:47,016 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:48,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:50,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:50,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:50,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:50,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:50,937 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:51,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:54,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:43:54,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:56,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:43:56,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:43:56,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:43:56,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:56,715 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:43:56,716 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:56,720 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:57,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:43:58,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:00,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:00,392 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:00,393 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:00,393 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:00,398 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:01,034 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:02,038 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:04,356 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:04,356 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:04,357 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:04,362 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:05,040 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:06,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:06,220 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:44:06,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:10,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:10,157 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:10,158 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:10,162 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:11,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:44:11,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:44:11,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:44:11,048 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:11,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:12,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:15,800 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:15,801 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:15,801 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:15,806 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:16,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:16,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:18,060 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:19,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:19,553 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:19,553 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:19,558 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:20,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:22,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:22,559 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:24,305 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:24,306 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:24,306 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:24,311 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:25,068 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:26,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:44:26,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:44:26,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:44:26,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:28,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:28,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:28,807 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:28,808 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:28,812 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:29,074 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:30,078 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:33,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:34,146 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:34,147 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:34,147 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:34,152 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:35,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:36,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:36,225 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:44:39,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:40,672 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:40,673 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:40,673 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:40,678 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:41,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:44:41,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:44:41,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:44:41,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:42,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:44,502 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:44,503 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:44,503 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:44,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:44,509 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:45,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:46,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:49,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:49,349 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:49,349 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:49,354 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:50,105 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:50,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:52,109 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:53,081 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:53,081 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:53,082 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:53,087 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:53,109 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:44:56,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:44:56,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:44:56,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:44:56,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:44:56,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:57,135 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:57,135 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:44:57,135 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:57,140 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:58,115 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:00,120 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:02,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:03,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:03,016 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:03,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:03,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:03,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:06,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:06,224 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:45:07,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:11,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:45:11,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:45:11,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:45:11,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:11,581 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:11,581 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:11,586 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:12,134 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:12,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:14,139 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:16,183 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:16,184 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:16,185 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:16,190 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:17,141 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:18,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:18,191 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:19,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:19,571 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:19,571 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:19,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:20,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:22,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:22,847 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:22,848 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:22,848 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:22,853 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:23,153 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:23,854 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:24,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:26,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:45:26,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:45:26,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:45:27,418 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:27,419 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:27,419 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:27,423 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:28,158 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:29,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:30,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:31,663 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:31,664 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:31,664 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:31,669 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:32,164 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:34,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:34,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:36,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:36,059 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:36,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:36,065 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:36,169 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:36,230 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:45:38,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:40,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:40,342 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:40,343 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:40,343 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:40,348 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:41,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:45:41,002 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:45:41,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:45:41,175 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:42,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:45,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:45,786 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:45,786 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:45,786 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:45,791 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:46,183 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:48,188 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:50,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:51,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:51,393 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:51,393 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:51,398 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:52,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:54,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:55,333 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:55,334 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:55,334 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:55,339 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:56,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:45:56,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:45:56,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:45:56,198 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:45:56,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:58,203 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:45:59,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:59,830 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:45:59,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:59,836 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:00,204 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:01,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:02,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:04,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:04,603 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:04,603 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:04,608 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:05,210 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:06,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:06,229 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:46:07,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:11,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:46:11,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:46:11,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:46:11,067 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:11,235 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:11,235 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:11,240 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:12,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:12,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:14,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:16,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:16,293 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:16,293 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:16,299 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:17,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:17,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:18,231 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:20,423 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:20,423 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:20,424 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:20,429 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:21,232 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:22,237 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:22,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:25,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:25,409 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:25,409 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:25,414 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:26,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:46:26,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:46:26,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:46:26,241 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:28,245 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:28,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:29,587 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:29,588 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:29,588 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:29,593 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:30,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:32,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:33,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:33,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:33,980 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:33,981 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:33,986 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:34,252 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:36,231 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:46:36,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:37,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:37,918 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:37,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:37,923 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:38,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:38,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:40,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:41,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:46:41,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:46:41,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:46:41,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:41,285 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:41,285 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:41,290 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:42,263 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:44,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:44,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:46,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:46,152 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:46,152 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:46,157 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:46,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:48,273 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:49,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:49,973 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:49,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:49,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:49,979 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:50,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:52,279 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:46:54,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:55,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:55,023 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:46:55,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:55,029 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:55,281 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:46:56,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:46:56,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:46:56,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:46:58,287 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:00,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:03,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:03,473 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:03,473 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:03,479 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:04,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:47:05,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:06,233 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:47:06,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:10,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:10,949 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:10,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:10,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:10,954 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:11,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:47:11,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:47:11,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:47:11,303 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:47:12,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:14,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:14,467 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:14,467 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:14,472 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:15,309 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:47:16,311 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:16,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:21,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:21,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:21,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:21,739 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:21,744 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:22,318 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:47:24,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:26,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:47:26,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:47:26,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:47:27,076 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:27,077 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:27,077 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:27,077 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:27,082 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:27,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:47:30,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:32,121 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:32,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:32,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:32,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:32,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:33,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:47:34,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:36,235 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:47:36,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:36,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:36,789 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:36,794 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:37,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:47:37,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:38,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:41,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:47:41,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:47:41,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:47:42,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:42,746 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:42,746 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:42,751 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:43,349 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:47:43,789 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:44,351 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:48,789 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:51,220 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:51,220 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:51,221 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:51,225 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:51,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:47:54,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:54,365 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:55,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:55,168 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:55,168 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:55,173 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:55,365 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:47:56,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:47:56,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:47:56,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:47:58,373 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:47:59,105 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:59,106 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:47:59,106 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:59,110 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:59,373 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:00,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:02,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:04,918 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:04,919 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:04,919 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:04,924 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:05,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:05,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:06,240 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:48:06,384 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:11,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:48:11,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:48:11,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:48:11,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:12,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:12,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:12,462 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:12,467 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:13,391 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:14,395 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:16,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:16,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:16,297 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:16,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:16,301 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:16,397 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:18,401 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:21,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:21,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:21,734 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:21,734 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:21,739 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:22,405 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:24,410 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:25,362 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:25,363 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:25,363 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:25,368 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:25,411 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:26,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:48:26,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:48:26,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:48:27,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:28,416 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:29,418 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:29,418 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:29,418 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:29,423 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:30,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:32,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:32,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:34,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:34,243 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:34,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:34,248 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:34,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:36,239 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:48:36,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:38,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:38,004 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:38,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:38,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:38,010 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:38,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:40,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:41,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:48:41,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:48:41,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:48:42,540 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:42,541 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:42,541 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:42,546 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:43,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:43,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:44,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:46,551 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:46,552 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:46,552 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:46,557 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:47,440 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:48,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:49,557 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:50,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:50,796 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:50,796 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:50,801 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:51,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:52,450 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:54,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:55,163 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:55,163 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:55,164 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:55,168 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:55,452 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:48:56,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:48:56,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:48:56,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:48:58,458 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:48:59,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:59,222 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:48:59,222 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:59,227 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:59,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:00,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:02,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:02,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:02,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:02,990 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:02,995 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:03,465 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:04,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:05,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:06,244 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:49:07,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:07,716 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:07,716 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:07,721 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:08,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:10,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:11,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:49:11,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:49:11,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:49:11,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:12,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:12,284 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:12,284 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:12,289 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:12,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:14,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:16,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:20,674 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:20,675 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:20,675 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:20,679 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:21,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:21,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:22,492 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:24,428 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:24,429 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:24,429 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:24,434 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:24,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:26,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:49:26,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:49:26,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:49:26,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:27,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:29,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:29,704 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:29,704 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:29,709 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:30,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:32,507 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:32,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:33,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:33,718 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:33,718 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:33,724 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:34,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:36,243 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:49:36,512 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:37,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:37,875 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:37,875 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:37,875 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:37,880 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:38,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:40,518 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:41,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:49:41,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:49:41,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:49:43,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:46,341 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:46,341 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:46,341 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:46,346 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:46,525 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:48,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:48,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:53,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:54,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:54,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:54,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:54,937 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:55,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:49:56,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:49:56,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:49:56,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:49:56,542 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:49:58,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:58,945 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:49:58,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:58,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:58,950 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:59,544 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:00,547 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:03,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:05,926 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:05,927 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:05,927 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:05,932 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:06,246 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:50:06,554 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:08,560 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:09,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:11,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:50:11,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:50:11,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:50:11,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:11,374 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:11,374 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:11,379 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:11,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:13,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:14,380 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:15,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:15,422 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:15,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:15,427 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:15,566 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:17,571 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:18,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:18,730 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:18,730 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:18,735 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:19,574 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:19,574 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:19,736 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:22,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:22,474 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:22,475 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:22,479 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:22,578 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:23,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:25,480 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:26,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:50:26,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:50:26,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:50:28,138 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:28,139 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:28,139 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:28,144 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:28,586 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:29,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:31,181 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:32,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:32,422 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:32,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:32,427 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:32,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:33,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:36,248 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:50:36,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:36,479 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:36,480 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:36,480 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:36,485 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:36,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:37,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:41,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:50:41,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:50:41,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:50:41,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:42,189 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:42,190 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:42,190 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:42,196 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:42,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:43,608 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:45,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:45,887 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:45,887 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:45,893 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:46,611 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:46,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:47,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:50,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:50,032 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:50,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:50,037 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:50,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:51,619 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:52,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:53,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:53,459 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:53,459 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:53,464 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:53,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:55,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:50:56,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:50:56,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:50:56,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:50:57,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:58,719 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:58,719 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:50:58,720 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:58,725 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:59,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:50:59,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:02,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:02,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:02,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:02,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:02,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:03,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:03,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:06,250 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:51:06,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:06,729 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:06,730 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:06,735 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:07,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:07,643 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:08,735 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:11,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:51:11,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:51:11,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:51:11,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:11,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:11,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:11,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:12,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:13,651 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:13,978 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:17,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:17,582 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:17,583 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:17,588 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:17,655 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:19,588 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:19,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:22,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:22,318 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:22,318 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:22,324 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:22,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:23,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:25,324 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:26,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:51:26,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:51:26,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:51:26,655 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:26,655 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:26,656 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:26,660 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:26,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:27,672 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:30,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:30,225 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:30,225 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:30,230 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:30,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:31,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:31,677 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:34,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:34,317 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:34,317 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:34,322 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:34,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:35,683 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:36,252 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:51:36,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:38,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:38,770 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:38,770 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:38,775 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:39,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:39,689 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:41,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:51:41,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:51:41,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:51:41,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:42,920 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:42,921 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:42,921 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:42,926 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:43,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:43,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:46,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:46,702 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:46,703 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:46,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:46,708 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:47,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:47,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:51,204 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:51,204 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:51,205 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:51,211 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:51,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:51,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:52,211 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:55,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:55,011 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:51:55,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:55,019 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:55,713 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:51:55,713 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:51:56,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:51:56,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:51:56,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:51:57,298 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:00,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:00,090 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:00,090 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:00,096 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:00,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:01,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:52:03,134 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:05,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:05,028 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:05,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:05,034 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:05,726 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:05,727 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:52:06,254 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:52:08,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:11,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:52:11,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:52:11,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:52:13,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:14,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:14,678 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:14,679 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:14,685 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:14,737 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:15,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:52:18,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:18,053 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:18,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:18,060 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:18,744 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:19,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:19,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:52:22,850 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:22,851 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:22,851 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:22,860 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:23,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:23,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:52:24,860 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:26,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:52:26,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:52:26,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:52:28,613 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:28,614 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:28,614 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:28,621 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:28,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:29,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:52:30,658 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:33,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:33,018 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:33,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:33,025 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:33,769 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:33,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:52:36,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:36,260 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:52:38,165 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:38,166 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:38,166 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:38,173 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:38,774 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:39,777 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:52:41,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:52:41,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:52:41,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:52:41,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:46,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:46,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:46,403 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:46,404 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:46,410 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:46,783 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:47,786 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:52:50,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:50,255 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:50,256 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:50,262 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:50,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:51,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:51,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:52:56,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:52:56,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:52:56,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:52:56,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:58,422 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:58,423 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:52:58,423 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:58,429 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:58,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:52:59,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:01,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:06,259 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:53:06,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:06,859 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:06,859 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:06,860 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:06,865 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:07,815 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:07,815 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:11,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:53:11,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:53:11,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:53:11,718 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:11,719 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:11,719 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:11,726 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:11,821 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:12,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:13,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:15,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:15,873 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:15,873 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:15,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:16,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:17,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:17,883 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:20,424 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:20,425 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:20,425 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:20,433 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:20,838 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:21,841 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:23,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:24,543 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:24,543 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:24,544 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:24,550 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:24,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:25,847 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:26,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:53:26,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:53:26,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:53:28,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:28,242 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:28,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:28,250 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:28,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:29,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:29,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:34,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:34,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:34,469 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:34,470 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:34,479 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:34,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:35,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:36,264 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:53:38,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:38,623 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:38,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:38,630 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:38,863 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:39,631 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:39,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:41,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:53:41,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:53:41,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:53:42,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:42,729 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:42,730 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:42,740 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:42,869 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:43,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:44,774 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:48,336 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:48,337 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:48,337 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:48,343 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:48,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:49,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:50,344 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:52,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:52,308 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:52,309 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:52,315 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:52,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:53,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:53:56,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:53:56,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:53:56,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:53:56,269 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:56,410 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:56,410 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:53:56,411 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:56,416 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:56,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:53:57,893 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:00,214 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:00,215 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:00,215 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:00,221 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:00,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:01,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:02,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:04,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:04,510 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:04,510 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:04,517 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:04,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:05,906 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:06,263 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:54:07,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:09,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:09,269 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:09,269 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:09,276 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:09,912 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:09,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:11,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:54:11,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:54:11,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:54:12,869 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:12,870 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:12,870 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:12,870 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:12,876 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:12,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:13,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:17,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:17,301 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:17,301 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:17,307 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:17,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:17,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:18,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:21,526 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:21,527 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:21,527 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:21,534 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:21,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:23,535 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:23,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:26,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:54:26,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:54:26,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:54:26,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:26,319 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:26,319 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:26,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:26,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:27,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:29,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:30,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:30,415 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:30,415 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:30,420 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:30,944 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:31,948 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:34,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:34,130 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:34,130 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:34,136 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:34,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:35,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:35,953 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:36,269 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:54:39,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:39,329 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:39,329 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:39,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:39,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:39,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:40,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:41,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:54:41,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:54:41,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:54:44,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:44,323 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:44,323 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:44,328 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:44,963 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:45,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:45,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:48,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:48,934 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:48,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:48,940 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:48,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:49,974 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:50,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:54,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:54,177 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:54:54,177 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:54,182 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:54,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:54:55,983 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:54:56,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:54:56,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:54:56,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:54:56,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:00,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:00,253 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:00,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:00,259 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:00,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:01,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:01,991 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:03,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:03,962 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:03,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:03,970 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:03,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:05,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:06,267 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:55:07,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:08,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:08,254 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:08,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:08,259 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:09,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:10,006 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:11,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:55:11,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:55:11,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:55:13,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:13,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:13,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:13,853 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:13,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:14,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:16,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:18,187 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:18,187 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:18,188 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:18,193 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:19,016 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:19,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:20,018 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:22,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:22,150 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:22,151 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:22,156 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:23,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:24,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:25,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:26,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:55:26,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:55:26,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:55:30,273 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:30,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:30,550 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:30,550 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:30,556 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:31,029 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:32,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:35,594 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:36,269 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:55:36,684 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:36,685 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:36,685 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:36,691 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:37,036 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:38,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:40,691 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:41,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:55:41,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:55:41,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:55:41,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:41,419 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:41,420 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:41,425 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:42,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:42,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:44,778 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:44,779 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:44,779 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:44,784 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:45,045 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:45,785 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:46,048 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:50,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:50,553 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:50,553 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:50,559 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:51,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:51,559 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:52,055 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:55,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:55,471 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:55:55,471 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:55,477 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:56,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:55:56,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:55:56,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:55:56,060 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:55:56,060 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:55:57,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:00,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:00,926 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:00,926 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:00,931 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:01,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:56:02,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:56:02,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:05,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:05,347 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:05,347 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:05,352 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:06,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:56:06,071 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:56:06,271 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:56:08,273 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:11,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:56:11,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:56:11,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:56:13,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:13,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:13,886 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:13,887 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:13,891 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:14,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:56:16,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:56:18,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:18,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:18,963 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:18,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:18,969 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:19,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:56:20,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:56:23,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:24,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:24,451 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:24,451 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:24,456 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:25,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:56:26,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:56:26,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:56:26,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:56:26,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:56:29,269 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:30,559 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:30,560 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:30,560 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:30,565 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:31,103 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:56:32,106 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:56:34,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:34,597 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:34,598 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:34,598 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:34,603 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:35,110 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:56:36,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:56:36,273 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:56:40,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:41,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:56:41,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:56:41,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:56:43,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:43,010 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:43,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:43,015 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:43,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:56:44,124 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:56:46,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:47,567 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:47,568 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:47,568 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:47,573 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:48,129 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:56:48,129 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:56:51,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:53,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:53,796 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:53,797 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:53,802 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:54,134 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:56:56,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:56:56,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:56:56,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:56:56,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:56:57,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:59,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:59,324 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:56:59,324 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:59,329 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:00,141 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:00,142 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:02,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:04,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:04,768 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:04,768 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:04,773 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:05,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:06,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:06,278 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:57:08,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:08,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:08,794 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:08,794 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:08,799 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:09,150 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:10,152 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:11,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:57:11,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:57:11,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:57:13,288 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:13,857 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:13,858 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:13,858 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:13,863 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:14,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:16,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:18,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:19,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:19,918 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:19,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:19,924 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:20,160 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:22,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:23,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:23,875 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:23,875 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:23,880 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:24,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:24,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:26,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:57:26,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:57:26,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:57:26,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:27,359 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:27,359 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:27,360 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:27,364 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:28,171 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:28,171 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:30,402 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:32,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:32,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:32,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:32,473 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:33,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:34,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:35,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:36,278 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:57:36,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:36,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:36,695 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:36,700 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:37,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:38,181 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:40,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:41,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:57:41,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:57:41,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:57:45,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:45,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:45,697 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:45,702 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:45,702 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:46,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:46,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:49,505 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:49,506 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:49,506 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:49,511 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:50,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:50,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:51,511 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:53,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:53,404 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:57:53,404 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:53,409 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:54,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:57:54,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:57:56,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:57:56,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:57:56,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:57:57,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:01,901 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:01,902 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:01,902 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:01,907 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:02,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:02,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:04,205 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:06,283 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:58:06,691 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:06,692 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:06,692 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:06,696 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:07,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:08,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:08,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:11,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:58:11,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:58:11,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:58:11,343 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:11,344 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:11,344 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:11,349 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:12,215 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:12,215 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:14,386 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:15,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:15,291 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:15,291 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:15,296 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:16,219 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:16,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:20,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:20,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:20,745 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:20,746 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:20,751 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:21,223 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:22,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:25,752 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:26,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:58:26,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:58:26,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:58:28,379 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:28,380 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:28,380 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:28,385 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:29,230 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:30,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:31,422 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:36,282 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:58:36,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:36,796 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:36,796 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:36,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:36,802 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:37,237 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:38,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:40,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:40,754 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:40,754 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:40,759 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:41,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:58:41,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:58:41,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:58:41,241 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:42,243 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:42,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:45,091 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:45,092 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:45,092 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:45,097 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:45,245 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:46,248 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:48,134 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:48,712 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:48,713 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:48,713 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:48,718 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:49,250 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:50,252 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:52,397 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:52,398 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:52,398 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:52,403 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:53,255 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:58:53,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:54,258 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:58:56,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:58:56,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:58:56,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:58:58,979 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:58,980 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:58:58,980 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:58,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:58,986 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:59,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:00,263 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:03,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:03,044 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:03,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:03,050 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:03,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:04,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:04,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:06,286 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:59:07,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:07,009 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:07,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:07,015 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:07,269 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:08,272 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:10,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:11,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:59:11,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:59:11,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:59:11,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:11,581 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:11,581 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:11,587 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:12,277 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:12,277 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:15,626 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:15,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:15,696 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:15,696 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:15,701 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:16,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:16,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:19,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:19,877 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:19,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:19,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:20,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:20,883 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:22,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:24,401 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:24,402 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:24,402 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:24,407 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:25,290 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:26,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:59:26,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:59:26,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:59:26,227 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:26,293 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:29,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:29,262 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:29,262 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:29,267 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:29,295 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:30,298 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:31,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:34,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:34,115 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:34,115 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:34,120 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:34,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:36,286 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 14:59:36,303 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:37,288 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:37,742 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:37,743 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:37,743 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:37,749 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:38,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:38,307 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:41,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:59:41,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:59:41,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:59:42,123 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:42,124 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:42,124 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:42,129 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:42,309 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:43,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:44,312 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:46,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:46,475 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:46,475 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:46,480 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:47,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:48,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:48,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:50,656 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:50,657 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:50,657 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:50,662 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:51,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:52,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:53,662 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:55,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:55,580 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 14:59:55,580 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:55,585 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:56,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:59:56,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:59:56,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:59:56,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 14:59:56,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 14:59:59,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:02,109 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:02,109 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:02,110 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:02,114 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:02,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:04,332 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:05,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:06,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:06,254 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:06,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:06,259 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:06,288 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:00:06,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:08,337 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:10,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:10,542 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:10,543 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:10,543 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:10,548 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:11,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:00:11,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:00:11,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:00:11,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:12,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:15,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:15,365 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:15,366 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:15,366 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:15,371 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:16,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:16,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:19,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:19,262 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:19,262 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:19,267 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:19,349 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:20,352 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:21,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:23,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:23,326 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:23,326 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:23,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:23,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:24,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:26,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:00:26,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:00:26,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:00:27,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:28,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:28,601 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:28,601 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:28,606 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:29,359 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:30,362 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:32,578 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:32,579 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:32,580 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:32,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:32,585 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:33,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:34,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:36,291 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:00:36,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:36,553 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:36,554 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:36,558 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:37,368 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:38,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:38,559 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:41,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:00:41,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:00:41,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:00:41,756 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:41,757 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:41,757 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:41,762 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:42,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:42,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:43,798 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:45,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:45,096 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:45,097 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:45,102 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:45,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:46,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:49,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:49,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:49,191 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:49,191 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:49,196 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:49,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:50,384 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:53,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:53,070 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:53,070 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:53,075 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:53,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:54,390 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:00:55,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:56,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:00:56,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:00:56,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:00:57,446 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:57,447 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:00:57,447 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:57,452 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:58,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:00:58,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:00,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:02,526 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:02,527 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:02,527 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:02,532 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:03,397 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:01:04,399 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:05,533 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:06,292 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:01:08,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:08,798 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:08,798 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:08,804 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:09,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:01:10,407 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:10,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:11,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:01:11,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:01:11,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:01:12,982 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:12,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:12,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:12,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:13,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:01:14,411 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:16,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:21,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:21,979 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:21,980 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:21,980 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:21,985 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:22,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:01:22,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:26,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:01:26,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:01:26,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:01:26,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:26,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:26,645 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:26,645 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:26,649 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:27,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:01:28,424 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:31,686 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:33,588 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:33,588 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:33,589 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:33,594 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:34,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:01:34,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:36,297 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:01:37,299 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:39,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:39,103 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:39,103 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:39,108 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:39,433 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:01:40,436 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:41,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:01:41,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:01:41,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:01:43,240 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:44,480 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:44,481 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:44,481 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:44,486 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:45,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:01:46,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:48,526 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:48,727 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:48,728 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:48,728 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:48,733 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:49,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:01:50,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:53,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:53,451 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:53,451 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:53,456 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:54,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:01:54,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:01:54,456 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:56,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:01:56,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:01:56,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:01:59,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:59,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:01:59,522 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:59,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:59,527 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:00,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:00,457 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:03,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:03,724 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:03,724 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:03,729 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:04,461 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:04,461 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:04,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:06,297 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:02:08,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:08,731 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:08,731 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:08,736 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:09,465 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:09,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:10,468 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:11,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:02:11,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:02:11,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:02:13,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:13,590 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:13,590 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:13,595 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:14,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:14,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:15,630 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:17,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:17,862 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:17,862 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:17,867 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:18,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:18,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:20,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:22,486 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:22,487 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:22,487 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:22,492 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:23,480 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:24,483 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:26,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:02:26,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:02:26,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:02:26,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:27,085 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:27,085 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:27,086 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:27,091 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:27,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:28,487 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:31,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:31,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:31,297 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:31,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:31,302 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:31,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:32,492 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:36,301 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:02:36,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:38,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:38,897 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:38,897 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:38,902 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:39,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:40,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:41,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:02:41,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:02:41,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:02:42,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:45,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:45,821 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:45,821 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:45,826 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:46,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:46,503 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:47,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:49,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:49,895 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:49,895 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:49,900 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:50,507 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:50,507 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:52,901 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:55,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:55,054 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:02:55,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:55,059 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:55,511 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:02:56,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:02:56,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:02:56,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:02:56,514 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:02:58,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:01,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:01,177 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:01,178 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:01,183 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:01,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:02,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:04,222 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:06,121 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:06,122 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:06,122 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:06,127 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:06,301 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:03:06,522 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:08,525 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:09,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:10,423 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:10,423 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:10,424 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:10,429 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:10,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:11,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:03:11,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:03:11,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:03:12,531 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:15,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:15,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:15,368 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:15,368 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:15,373 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:15,533 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:16,536 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:19,571 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:19,572 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:19,572 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:19,578 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:20,540 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:20,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:20,578 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:24,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:24,490 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:24,491 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:24,496 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:24,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:26,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:03:26,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:03:26,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:03:26,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:26,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:28,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:28,750 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:28,750 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:28,755 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:29,548 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:30,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:31,794 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:36,306 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:03:37,110 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:37,111 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:37,111 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:37,111 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:37,116 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:37,554 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:38,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:41,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:03:41,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:03:41,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:03:41,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:41,835 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:41,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:41,840 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:42,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:42,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:42,878 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:45,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:45,963 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:45,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:45,968 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:46,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:46,566 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:47,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:49,493 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:49,494 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:49,494 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:49,499 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:49,567 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:50,570 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:53,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:53,724 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:53,725 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:53,725 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:53,731 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:54,575 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:54,575 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:56,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:03:56,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:03:56,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:03:57,143 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:57,144 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:03:57,144 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:57,150 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:57,577 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:03:58,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:03:59,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:03,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:03,435 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:03,435 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:03,440 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:03,582 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:04,440 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:04,585 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:06,305 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:04:08,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:08,272 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:08,272 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:08,277 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:08,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:10,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:10,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:11,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:04:11,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:04:11,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:04:14,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:14,810 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:14,810 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:14,815 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:15,595 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:15,854 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:16,598 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:20,854 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:23,418 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:23,419 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:23,419 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:23,424 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:23,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:24,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:26,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:04:26,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:04:26,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:04:26,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:26,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:26,874 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:26,874 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:26,879 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:27,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:28,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:30,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:30,730 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:30,730 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:30,735 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:31,611 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:31,736 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:32,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:36,307 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:04:36,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:36,749 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:36,750 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:36,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:36,755 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:37,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:38,619 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:40,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:40,591 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:40,591 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:40,596 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:40,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:41,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:04:41,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:04:41,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:04:42,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:42,624 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:44,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:44,431 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:44,431 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:44,436 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:44,625 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:46,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:47,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:48,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:48,622 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:48,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:48,627 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:48,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:50,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:52,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:52,571 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:52,571 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:52,571 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:52,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:52,636 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:54,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:04:56,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:04:56,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:04:56,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:04:57,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:57,176 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:04:57,176 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:57,181 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:57,641 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:04:58,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:58,643 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:00,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:00,533 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:00,533 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:00,538 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:00,645 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:02,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:03,539 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:05,144 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:05,145 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:05,145 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:05,150 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:05,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:06,309 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:05:06,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:09,311 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:09,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:09,948 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:09,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:09,954 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:10,658 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:10,658 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:11,008 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:05:11,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:05:11,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:05:15,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:16,488 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:16,489 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:16,489 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:16,494 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:16,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:18,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:20,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:20,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:20,326 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:20,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:20,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:20,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:22,670 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:25,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:26,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:05:26,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:05:26,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:05:28,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:28,608 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:28,608 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:28,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:28,674 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:30,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:30,677 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:32,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:32,134 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:32,134 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:32,139 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:32,680 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:32,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:36,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:36,090 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:36,090 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:36,091 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:36,096 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:36,312 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:05:36,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:36,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:41,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:05:41,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:05:41,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:05:41,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:41,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:41,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:41,808 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:41,813 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:42,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:42,691 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:46,850 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:49,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:49,774 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:49,774 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:49,779 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:50,697 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:50,697 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:52,779 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:54,487 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:54,488 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:54,488 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:54,493 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:54,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:05:56,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:05:56,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:05:56,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:05:56,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:05:58,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:58,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:58,309 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:05:58,309 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:58,314 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:58,705 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:00,708 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:03,351 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:03,351 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:03,352 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:03,352 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:03,357 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:03,710 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:04,713 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:06,314 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:06:08,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:08,795 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:08,796 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:08,796 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:08,801 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:09,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:10,719 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:11,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:06:11,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:06:11,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:06:14,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:14,275 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:14,275 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:14,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:14,280 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:14,724 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:14,724 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:18,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:18,266 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:18,267 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:18,271 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:18,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:18,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:20,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:22,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:22,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:22,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:22,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:22,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:24,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:25,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:26,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:06:26,009 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:06:26,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:06:26,612 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:26,613 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:26,613 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:26,618 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:26,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:28,739 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:30,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:33,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:33,054 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:33,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:33,059 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:33,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:34,745 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:36,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:36,316 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:06:41,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:06:41,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:06:41,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:06:41,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:42,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:42,291 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:42,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:42,296 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:42,751 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:42,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:46,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:47,571 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:47,572 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:47,572 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:47,577 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:47,754 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:48,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:51,578 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:51,592 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:51,593 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:51,593 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:51,598 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:51,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:52,762 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:06:56,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:06:56,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:06:56,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:06:57,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:57,445 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:57,445 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:06:57,446 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:57,450 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:57,765 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:06:58,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:02,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:04,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:04,143 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:04,143 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:04,148 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:04,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:04,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:06,328 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:07:07,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:07,793 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:07,794 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:07,794 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:07,798 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:08,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:08,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:11,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:07:11,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:07:11,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:07:11,869 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:11,869 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:11,870 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:11,875 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:12,783 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:12,783 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:12,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:16,559 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:16,559 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:16,560 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:16,565 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:16,785 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:18,565 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:18,789 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:19,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:19,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:19,990 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:19,995 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:20,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:20,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:23,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:25,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:25,356 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:25,356 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:25,361 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:25,795 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:26,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:07:26,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:07:26,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:07:26,798 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:29,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:33,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:33,811 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:33,811 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:33,816 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:34,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:34,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:34,854 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:36,320 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:07:37,987 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:37,988 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:37,988 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:37,993 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:38,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:38,810 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:39,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:41,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:07:41,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:07:41,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:07:44,415 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:44,416 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:44,416 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:44,421 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:44,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:45,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:46,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:47,756 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:47,757 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:47,757 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:47,762 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:47,817 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:48,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:50,763 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:51,675 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:51,676 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:51,676 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:51,681 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:51,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:52,824 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:55,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:55,578 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:55,578 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:55,584 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:55,826 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:07:56,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:07:56,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:07:56,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:07:56,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:56,829 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:07:58,883 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:58,884 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:07:58,884 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:58,889 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:59,831 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:00,833 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:01,926 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:03,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:03,214 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:03,214 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:03,219 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:03,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:04,837 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:06,325 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:08:07,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:09,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:09,027 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:09,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:09,032 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:09,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:10,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:11,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:08:11,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:08:11,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:08:13,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:13,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:13,270 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:13,271 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:13,275 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:13,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:14,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:18,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:19,188 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:19,188 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:19,189 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:19,193 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:19,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:20,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:23,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:23,605 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:23,606 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:23,606 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:23,611 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:23,856 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:24,859 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:26,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:08:26,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:08:26,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:08:28,530 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:28,530 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:28,530 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:28,535 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:28,862 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:29,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:30,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:32,396 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:32,396 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:32,397 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:32,402 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:32,868 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:32,868 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:35,402 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:36,324 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:08:36,983 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:36,984 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:36,984 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:36,989 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:37,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:38,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:40,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:41,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:08:41,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:08:41,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:08:41,491 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:41,491 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:41,492 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:41,496 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:41,875 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:42,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:45,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:45,642 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:45,642 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:45,647 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:45,879 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:46,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:46,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:51,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:54,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:54,025 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:54,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:54,030 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:54,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:08:54,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:08:56,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:08:56,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:08:56,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:08:57,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:58,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:58,577 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:08:58,577 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:58,582 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:58,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:00,895 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:02,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:02,911 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:02,912 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:02,913 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:02,917 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:03,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:04,900 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:06,326 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:09:06,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:06,867 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:06,867 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:06,872 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:06,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:07,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:08,904 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:11,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:11,006 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:11,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:11,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:09:11,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:09:11,011 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:11,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:09:11,907 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:12,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:13,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:14,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:14,964 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:14,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:14,968 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:15,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:16,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:18,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:18,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:18,808 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:18,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:18,813 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:18,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:20,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:22,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:22,966 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:22,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:22,971 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:23,920 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:23,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:24,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:26,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:09:26,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:09:26,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:09:26,702 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:26,703 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:26,703 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:26,707 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:26,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:28,926 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:29,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:34,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:35,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:35,243 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:35,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:35,248 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:35,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:36,328 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:09:36,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:38,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:38,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:38,853 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:38,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:38,934 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:39,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:40,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:41,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:09:41,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:09:41,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:09:43,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:43,557 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:43,557 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:43,562 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:43,939 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:44,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:45,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:47,402 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:47,403 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:47,404 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:47,409 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:47,943 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:48,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:51,409 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:52,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:52,030 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:52,030 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:52,035 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:52,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:52,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:09:56,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:09:56,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:09:56,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:09:56,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:56,821 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:09:56,822 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:56,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:56,827 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:56,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:09:58,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:01,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:03,092 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:03,093 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:03,093 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:03,099 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:03,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:04,963 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:06,330 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:10:07,116 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:07,117 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:07,117 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:07,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:07,122 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:07,965 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:08,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:10,860 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:10,861 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:10,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:10,866 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:10,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:11,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:10:11,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:10:11,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:10:12,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:12,973 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:17,110 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:17,110 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:17,111 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:17,116 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:17,976 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:18,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:18,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:21,222 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:21,223 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:21,223 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:21,228 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:21,981 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:22,983 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:23,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:26,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:10:26,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:10:26,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:10:28,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:28,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:28,967 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:28,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:28,972 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:28,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:30,990 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:34,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:34,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:34,131 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:34,131 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:34,136 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:34,995 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:34,995 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:36,333 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:10:39,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:40,446 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:40,447 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:40,447 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:40,452 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:41,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:41,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:41,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:10:41,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:10:41,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:10:45,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:46,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:46,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:46,296 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:46,301 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:47,006 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:47,006 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:50,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:51,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:51,029 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:51,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:51,034 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:52,010 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:53,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:55,869 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:55,870 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:55,870 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:55,870 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:55,875 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:56,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:10:56,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:10:56,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:10:56,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:10:57,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:10:59,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:59,422 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:10:59,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:59,427 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:00,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:01,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:01,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:04,391 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:04,392 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:04,392 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:04,397 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:05,026 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:05,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:06,335 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:11:07,336 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:09,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:09,758 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:09,758 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:09,763 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:10,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:11,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:11:11,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:11:11,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:11:11,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:13,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:18,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:18,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:18,356 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:18,356 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:18,361 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:19,040 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:19,040 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:22,356 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:22,357 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:22,357 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:22,364 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:23,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:23,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:23,364 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:26,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:11:26,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:11:26,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:11:28,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:28,051 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:28,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:28,056 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:29,050 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:29,050 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:29,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:32,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:32,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:32,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:32,988 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:33,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:34,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:35,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:36,340 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:11:36,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:36,887 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:36,887 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:36,892 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:37,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:39,060 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:40,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:40,272 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:40,272 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:40,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:40,277 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:41,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:11:41,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:11:41,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:11:41,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:41,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:44,491 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:44,491 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:44,492 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:44,497 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:45,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:45,068 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:45,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:48,371 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:48,372 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:48,372 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:48,377 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:49,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:49,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:51,378 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:52,420 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:52,421 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:52,421 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:52,426 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:53,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:53,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:11:56,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:11:56,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:11:56,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:11:56,659 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:56,659 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:11:56,660 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:56,660 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:56,664 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:57,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:11:57,082 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:00,637 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:00,638 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:00,638 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:00,643 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:01,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:01,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:02,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:06,339 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:12:07,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:07,286 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:07,286 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:07,291 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:08,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:08,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:09,092 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:11,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:12:11,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:12:11,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:12:11,652 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:11,652 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:11,652 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:11,657 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:12,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:13,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:13,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:15,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:15,547 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:15,547 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:15,552 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:16,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:17,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:19,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:20,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:20,242 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:20,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:20,247 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:21,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:21,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:24,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:24,030 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:24,030 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:24,035 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:24,109 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:25,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:25,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:26,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:12:26,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:12:26,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:12:29,836 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:29,836 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:29,837 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:29,842 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:30,115 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:30,878 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:31,118 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:34,396 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:34,397 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:34,397 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:34,402 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:35,122 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:35,122 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:36,344 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:12:36,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:38,386 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:38,386 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:38,387 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:38,392 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:39,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:39,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:41,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:12:41,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:12:41,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:12:42,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:46,573 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:46,574 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:46,574 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:46,579 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:47,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:47,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:47,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:51,727 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:51,727 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:51,728 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:51,733 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:52,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:52,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:53,139 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:56,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:12:56,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:12:56,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:12:56,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:56,314 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:12:56,314 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:56,319 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:57,143 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:12:57,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:12:58,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:00,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:00,040 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:00,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:00,046 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:00,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:01,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:04,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:04,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:04,320 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:04,320 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:04,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:05,150 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:05,150 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:06,343 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:13:08,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:08,874 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:08,874 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:08,880 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:09,153 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:09,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:11,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:13:11,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:13:11,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:13:11,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:12,878 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:12,879 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:12,879 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:12,884 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:13,158 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:14,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:15,161 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:19,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:21,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:21,163 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:21,163 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:21,168 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:21,169 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:23,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:25,169 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:26,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:13:26,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:13:26,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:13:26,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:26,953 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:26,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:26,958 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:27,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:29,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:30,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:32,531 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:32,532 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:32,532 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:32,537 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:33,182 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:33,182 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:36,345 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:13:36,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:36,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:36,365 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:36,365 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:36,370 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:37,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:37,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:41,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:13:41,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:13:41,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:13:41,150 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:41,269 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:41,270 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:41,275 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:42,189 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:42,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:43,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:45,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:45,335 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:45,335 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:45,340 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:46,193 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:47,196 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:47,340 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:50,318 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:50,318 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:50,319 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:50,327 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:51,200 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:51,200 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:53,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:54,104 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:54,105 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:54,105 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:54,110 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:54,203 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:55,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:13:56,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:13:56,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:13:56,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:13:57,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:57,850 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:13:57,850 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:57,855 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:58,207 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:13:58,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:59,210 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:01,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:01,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:01,808 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:01,814 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:02,212 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:03,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:04,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:06,347 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:14:07,343 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:07,344 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:07,345 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:07,350 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:08,217 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:09,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:10,350 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:11,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:14:11,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:14:11,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:14:13,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:13,590 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:13,590 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:13,595 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:14,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:15,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:15,634 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:17,396 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:17,396 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:17,397 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:17,401 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:18,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:19,232 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:21,168 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:21,169 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:21,169 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:21,169 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:21,174 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:21,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:23,236 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:25,771 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:25,771 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:25,772 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:25,777 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:26,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:14:26,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:14:26,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:14:26,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:26,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:27,241 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:30,705 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:30,706 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:30,706 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:30,711 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:31,245 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:31,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:31,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:35,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:35,478 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:35,478 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:35,483 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:36,248 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:36,349 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:14:37,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:37,351 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:39,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:39,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:39,522 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:39,527 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:40,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:41,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:14:41,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:14:41,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:14:41,255 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:43,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:43,236 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:43,237 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:43,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:43,242 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:43,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:45,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:48,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:51,643 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:51,644 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:51,644 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:51,649 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:52,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:53,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:14:53,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:56,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:14:56,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:14:56,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:14:57,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:57,683 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:14:57,683 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:57,688 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:58,270 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:14:58,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:59,273 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:03,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:05,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:05,832 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:05,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:05,837 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:06,277 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:15:06,352 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:15:07,279 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:09,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:10,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:10,404 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:10,404 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:10,409 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:11,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:15:11,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:15:11,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:15:11,285 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:15:11,285 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:14,390 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:14,391 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:14,391 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:14,391 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:14,396 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:15,289 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:15:15,289 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:17,688 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:17,689 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:17,690 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:17,695 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:18,292 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:15:19,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:19,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:24,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:26,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:15:26,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:15:26,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:15:26,350 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:26,351 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:26,351 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:26,356 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:27,301 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:15:27,301 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:30,394 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:31,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:31,995 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:31,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:32,001 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:32,304 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:15:33,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:35,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:35,810 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:35,810 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:35,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:35,816 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:36,308 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:15:36,354 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:15:37,311 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:40,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:40,553 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:40,554 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:40,559 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:41,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:15:41,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:15:41,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:15:41,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:41,315 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:15:41,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:46,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:49,191 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:49,192 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:49,192 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:49,197 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:49,320 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:15:51,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:52,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:53,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:53,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:53,699 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:53,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:54,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:15:55,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:15:56,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:15:56,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:15:56,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:15:57,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:59,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:59,383 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:15:59,383 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:59,388 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:00,332 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:01,335 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:02,426 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:03,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:03,332 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:03,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:03,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:03,338 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:05,341 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:06,359 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:16:08,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:08,042 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:08,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:08,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:08,048 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:08,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:09,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:11,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:16:11,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:16:11,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:16:13,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:14,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:14,050 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:14,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:14,055 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:14,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:15,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:18,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:18,958 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:18,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:18,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:18,964 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:19,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:19,355 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:22,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:22,860 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:22,860 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:22,865 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:23,359 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:23,359 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:24,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:26,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:16:26,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:16:26,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:16:26,640 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:26,640 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:26,641 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:26,645 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:27,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:27,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:30,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:34,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:34,998 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:34,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:35,003 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:35,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:35,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:36,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:36,358 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:16:38,740 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:38,740 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:38,741 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:38,746 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:39,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:39,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:41,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:16:41,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:16:41,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:16:41,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:43,183 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:43,184 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:43,184 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:43,189 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:43,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:45,380 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:46,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:46,956 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:46,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:46,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:46,961 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:47,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:47,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:50,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:50,996 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:50,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:51,001 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:51,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:51,388 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:52,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:55,889 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:55,890 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:55,890 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:55,895 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:56,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:16:56,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:16:56,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:16:56,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:16:57,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:57,395 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:16:59,906 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:59,906 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:16:59,906 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:59,911 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:00,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:01,399 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:02,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:04,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:04,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:04,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:04,474 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:05,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:05,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:06,363 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:17:08,364 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:08,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:08,592 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:08,592 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:08,597 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:09,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:09,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:11,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:17:11,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:17:11,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:17:14,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:14,431 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:14,432 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:14,432 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:14,437 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:15,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:15,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:19,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:20,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:20,043 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:20,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:20,048 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:20,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:21,420 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:24,512 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:24,513 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:24,513 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:24,513 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:24,518 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:25,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:25,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:26,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:17:26,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:17:26,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:17:28,311 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:28,312 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:28,312 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:28,317 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:28,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:29,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:30,354 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:31,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:31,938 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:31,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:31,943 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:32,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:33,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:35,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:36,362 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:17:37,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:37,349 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:37,349 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:37,354 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:37,437 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:39,440 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:41,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:17:41,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:17:41,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:17:41,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:43,594 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:43,595 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:43,595 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:43,600 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:44,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:45,445 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:46,638 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:48,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:48,354 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:48,354 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:48,359 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:48,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:49,450 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:52,360 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:54,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:54,945 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:17:54,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:54,950 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:55,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:17:55,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:17:56,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:17:56,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:17:56,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:17:58,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:00,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:00,724 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:00,724 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:00,729 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:01,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:01,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:18:03,766 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:04,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:04,571 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:04,571 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:04,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:05,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:05,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:18:06,367 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:18:09,369 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:10,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:10,633 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:10,634 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:10,639 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:11,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:18:11,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:18:11,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:18:11,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:11,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:18:15,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:16,086 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:16,087 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:16,087 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:16,092 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:16,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:17,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:18:21,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:22,138 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:22,139 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:22,139 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:22,144 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:22,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:23,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:18:26,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:18:26,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:18:26,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:18:26,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:27,666 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:27,666 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:27,666 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:27,672 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:28,487 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:29,490 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:18:31,710 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:36,366 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:18:36,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:36,903 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:36,903 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:36,903 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:36,909 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:37,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:37,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:18:41,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:18:41,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:18:41,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:18:41,572 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:41,572 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:41,572 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:41,577 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:42,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:42,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:43,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:18:46,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:46,649 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:46,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:46,655 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:47,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:47,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:18:47,655 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:51,399 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:51,400 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:51,400 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:51,405 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:51,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:53,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:53,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:18:56,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:18:56,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:18:56,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:18:57,161 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:57,161 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:18:57,161 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:57,166 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:57,515 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:18:59,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:59,518 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:02,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:02,157 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:02,157 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:02,162 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:02,520 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:03,522 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:05,163 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:05,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:05,731 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:05,731 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:05,736 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:06,368 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:19:06,524 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:07,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:09,411 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:09,412 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:09,412 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:09,417 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:09,528 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:10,417 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:11,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:19:11,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:19:11,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:19:11,532 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:14,132 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:14,133 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:14,133 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:14,137 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:14,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:15,537 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:16,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:18,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:18,320 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:18,320 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:18,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:18,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:19,542 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:21,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:23,871 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:23,872 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:23,872 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:23,877 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:24,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:25,547 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:26,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:19:26,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:19:26,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:19:27,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:32,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:32,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:32,285 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:32,285 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:32,290 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:32,551 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:33,554 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:36,370 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:19:36,592 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:36,593 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:36,593 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:36,598 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:37,559 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:37,559 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:37,599 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:40,709 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:40,710 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:40,710 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:40,715 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:41,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:19:41,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:19:41,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:19:41,563 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:41,563 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:43,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:44,603 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:44,604 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:44,604 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:44,609 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:45,567 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:45,567 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:48,646 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:48,664 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:48,665 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:48,665 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:48,670 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:49,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:49,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:53,671 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:56,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:19:56,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:19:56,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:19:57,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:57,056 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:19:57,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:57,061 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:57,577 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:19:57,577 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:19:59,098 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:01,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:01,239 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:01,239 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:01,244 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:01,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:01,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:04,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:05,124 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:05,125 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:05,125 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:05,130 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:05,585 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:05,586 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:06,372 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:20:09,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:10,495 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:10,496 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:10,496 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:10,501 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:10,591 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:11,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:20:11,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:20:11,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:20:11,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:15,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:15,152 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:15,152 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:15,152 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:15,157 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:15,598 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:15,598 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:19,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:19,103 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:19,103 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:19,108 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:19,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:19,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:21,109 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:23,080 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:23,081 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:23,081 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:23,086 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:23,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:23,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:26,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:20:26,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:20:26,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:20:26,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:28,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:28,608 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:28,608 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:28,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:28,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:29,616 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:31,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:33,436 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:33,437 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:33,438 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:33,443 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:33,618 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:35,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:36,378 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:20:37,379 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:41,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:20:41,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:20:41,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:20:42,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:42,090 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:42,090 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:42,095 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:42,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:43,134 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:43,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:46,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:46,562 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:46,562 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:46,567 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:46,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:47,632 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:48,567 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:50,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:50,516 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:50,517 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:50,521 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:50,634 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:51,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:54,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:54,518 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:54,518 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:54,518 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:54,523 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:54,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:55,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:20:56,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:20:56,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:20:56,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:20:58,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:58,518 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:20:58,518 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:58,523 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:58,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:20:59,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:59,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:02,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:02,630 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:02,630 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:02,635 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:02,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:03,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:04,636 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:06,377 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:21:08,088 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:08,089 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:08,089 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:08,094 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:08,653 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:09,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:10,095 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:11,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:21:11,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:21:11,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:21:12,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:12,264 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:12,264 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:12,269 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:12,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:13,662 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:15,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:16,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:16,509 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:16,510 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:16,515 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:16,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:17,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:20,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:21,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:21,699 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:21,699 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:21,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:22,670 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:23,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:25,705 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:26,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:21:26,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:21:26,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:21:27,424 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:27,425 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:27,425 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:27,430 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:27,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:29,678 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:31,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:35,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:35,973 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:35,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:35,978 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:36,382 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:21:36,682 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:37,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:37,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:39,785 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:39,786 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:39,786 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:39,791 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:40,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:41,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:21:41,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:21:41,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:21:41,689 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:43,303 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:43,864 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:43,865 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:43,865 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:43,871 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:44,691 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:45,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:47,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:47,585 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:47,585 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:47,590 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:47,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:48,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:49,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:51,140 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:51,141 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:51,141 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:51,146 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:51,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:51,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:54,147 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:55,786 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:55,787 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:55,787 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:55,792 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:56,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:21:56,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:21:56,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:21:56,705 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:21:57,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:21:59,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:59,397 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:59,397 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:21:59,398 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:59,403 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:59,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:01,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:03,613 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:03,614 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:03,614 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:03,619 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:03,713 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:04,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:05,716 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:06,381 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:22:07,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:07,768 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:07,768 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:07,773 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:08,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:09,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:09,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:11,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:22:11,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:22:11,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:22:11,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:11,806 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:11,806 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:11,811 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:12,724 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:13,726 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:14,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:16,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:16,038 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:16,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:16,044 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:16,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:17,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:20,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:21,377 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:21,378 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:21,378 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:21,383 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:21,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:21,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:25,200 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:25,201 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:25,201 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:25,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:25,206 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:25,740 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:25,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:26,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:22:26,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:22:26,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:22:30,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:30,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:30,989 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:30,989 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:30,994 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:31,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:31,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:35,812 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:35,813 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:35,813 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:35,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:35,818 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:36,386 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:22:36,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:37,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:41,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:22:41,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:22:41,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:22:41,159 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:41,287 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:41,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:41,287 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:41,292 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:41,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:41,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:44,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:44,872 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:44,873 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:44,879 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:45,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:45,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:46,879 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:48,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:48,266 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:48,266 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:48,271 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:48,762 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:49,765 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:52,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:52,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:52,937 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:52,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:52,942 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:53,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:53,769 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:56,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:22:56,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:22:56,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:22:56,878 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:56,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:22:56,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:56,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:57,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:22:57,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:22:57,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:01,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:01,821 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:01,821 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:01,826 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:02,776 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:03,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:03,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:06,385 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:23:07,720 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:07,721 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:07,721 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:07,726 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:07,781 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:09,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:09,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:11,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:23:11,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:23:11,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:23:13,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:13,474 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:13,474 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:13,479 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:13,787 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:15,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:15,791 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:17,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:17,504 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:17,504 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:17,509 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:17,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:19,796 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:21,072 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:21,072 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:21,073 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:21,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:21,078 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:21,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:21,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:26,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:23:26,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:23:26,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:23:26,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:26,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:26,422 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:26,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:26,427 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:26,803 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:27,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:31,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:31,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:31,651 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:31,651 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:31,656 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:31,808 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:33,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:36,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:36,031 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:36,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:36,037 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:36,391 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:23:36,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:37,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:37,815 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:41,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:23:41,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:23:41,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:23:42,610 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:42,611 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:42,611 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:42,611 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:42,617 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:42,818 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:43,821 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:47,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:47,014 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:47,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:47,019 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:47,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:47,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:48,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:50,586 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:50,586 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:50,587 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:50,592 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:50,827 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:51,830 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:53,592 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:54,604 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:54,605 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:23:54,605 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:54,610 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:54,833 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:23:55,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:23:56,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:23:56,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:23:56,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:23:59,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:02,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:02,960 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:02,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:02,965 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:03,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:03,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:24:05,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:06,390 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:24:07,169 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:07,170 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:07,170 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:07,175 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:07,847 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:07,847 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:24:10,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:11,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:24:11,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:24:11,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:24:11,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:11,318 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:11,318 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:11,323 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:11,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:11,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:24:15,324 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:20,099 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:20,099 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:20,100 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:20,105 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:20,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:21,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:21,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:24:26,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:24:26,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:24:26,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:24:26,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:29,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:29,375 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:29,375 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:29,380 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:29,868 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:29,868 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:24:31,417 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:33,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:33,049 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:33,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:33,055 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:33,872 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:33,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:24:36,392 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:24:37,393 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:37,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:37,547 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:37,547 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:37,552 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:37,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:37,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:24:41,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:24:41,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:24:41,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:24:41,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:41,683 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:41,684 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:41,688 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:41,879 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:42,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:43,883 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:24:47,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:50,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:50,098 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:50,098 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:50,103 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:50,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:51,889 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:24:53,104 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:53,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:53,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:53,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:53,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:53,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:55,895 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:24:56,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:24:56,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:24:56,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:24:58,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:58,705 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:58,708 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:24:58,709 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:58,714 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:58,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:24:59,900 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:02,771 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:02,772 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:02,773 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:02,778 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:02,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:03,778 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:03,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:06,394 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:25:07,480 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:07,481 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:07,481 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:07,486 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:07,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:07,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:09,487 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:11,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:25:11,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:25:11,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:25:12,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:12,535 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:12,535 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:12,540 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:12,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:13,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:14,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:16,118 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:16,119 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:16,120 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:16,125 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:16,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:17,920 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:20,126 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:20,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:20,944 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:20,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:20,950 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:21,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:21,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:24,743 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:24,744 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:24,745 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:24,750 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:24,926 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:25,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:25,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:26,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:25:26,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:25:26,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:25:29,709 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:29,710 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:29,710 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:29,715 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:29,932 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:30,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:31,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:35,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:36,396 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:25:36,464 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:36,465 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:36,465 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:36,470 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:36,938 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:37,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:41,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:25:41,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:25:41,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:25:41,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:41,233 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:41,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:41,233 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:41,238 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:41,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:41,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:46,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:48,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:48,011 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:48,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:48,016 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:48,949 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:49,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:52,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:52,594 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:52,595 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:52,595 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:52,600 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:52,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:53,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:56,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:25:56,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:25:56,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:25:56,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:56,943 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:25:56,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:56,948 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:56,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:25:57,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:25:57,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:02,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:05,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:05,475 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:05,475 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:05,480 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:05,968 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:05,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:06,401 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:26:08,402 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:09,661 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:09,662 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:09,662 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:09,667 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:09,973 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:09,973 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:11,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:26:11,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:26:11,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:26:13,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:13,392 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:13,393 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:13,398 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:13,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:13,978 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:13,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:18,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:19,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:19,219 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:19,219 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:19,224 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:19,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:19,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:23,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:23,034 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:23,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:23,039 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:23,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:23,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:24,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:26,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:26:26,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:26:26,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:26:27,400 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:27,400 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:27,401 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:27,405 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:27,994 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:27,994 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:29,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:31,340 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:31,340 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:31,341 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:31,346 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:31,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:31,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:35,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:35,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:35,580 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:35,580 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:35,585 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:36,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:36,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:36,400 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:26:39,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:39,408 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:39,409 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:39,414 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:40,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:40,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:40,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:41,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:26:41,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:26:41,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:26:43,357 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:43,358 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:43,358 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:43,363 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:44,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:44,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:46,401 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:48,193 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:48,194 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:48,194 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:48,199 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:49,014 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:50,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:52,199 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:52,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:52,580 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:52,580 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:52,585 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:53,020 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:54,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:26:56,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:26:56,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:26:56,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:26:57,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:57,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:57,580 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:26:57,580 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:57,585 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:58,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:26:58,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:02,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:04,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:04,307 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:04,307 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:04,313 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:05,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:06,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:06,404 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:27:08,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:08,431 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:08,432 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:08,432 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:08,438 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:09,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:10,038 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:11,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:27:11,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:27:11,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:27:12,613 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:12,614 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:12,614 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:12,619 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:13,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:13,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:14,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:16,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:16,626 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:16,627 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:16,632 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:17,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:18,049 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:19,632 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:20,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:20,897 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:20,897 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:20,902 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:21,051 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:22,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:24,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:24,746 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:24,747 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:24,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:24,753 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:25,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:26,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:27:26,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:27:26,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:27:26,059 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:29,157 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:29,157 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:29,158 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:29,163 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:30,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:30,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:30,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:32,899 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:32,900 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:32,900 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:32,906 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:33,065 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:34,068 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:35,906 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:36,404 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:27:37,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:37,230 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:37,230 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:37,236 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:38,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:38,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:41,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:27:41,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:27:41,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:27:41,200 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:41,263 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:41,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:41,263 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:41,269 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:42,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:42,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:45,843 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:45,843 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:45,843 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:45,849 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:46,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:46,885 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:48,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:49,311 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:49,312 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:49,312 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:49,317 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:50,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:50,087 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:27:52,318 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:56,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:27:56,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:27:56,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:27:56,451 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:56,452 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:27:56,452 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:56,457 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:57,092 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:27:57,493 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:58,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:01,635 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:01,635 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:01,636 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:01,641 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:02,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:02,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:02,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:05,387 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:05,388 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:05,389 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:05,394 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:06,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:06,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:06,409 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:28:08,411 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:09,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:09,570 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:09,571 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:09,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:10,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:10,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:11,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:28:11,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:28:11,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:28:14,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:14,206 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:14,206 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:14,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:14,211 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:15,111 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:16,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:18,839 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:18,839 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:18,840 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:18,845 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:19,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:19,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:20,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:24,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:26,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:28:26,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:28:26,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:28:27,452 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:27,452 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:27,453 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:27,458 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:28,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:28,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:30,493 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:33,540 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:33,541 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:33,541 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:33,546 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:34,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:34,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:35,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:36,408 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:28:38,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:38,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:38,696 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:38,701 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:39,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:40,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:40,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:41,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:28:41,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:28:41,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:28:46,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:46,207 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:46,207 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:46,207 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:46,212 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:47,141 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:48,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:50,121 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:50,122 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:50,122 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:50,127 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:50,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:52,127 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:52,149 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:54,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:54,420 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:28:54,420 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:54,425 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:55,152 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:28:56,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:28:56,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:28:56,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:28:56,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:28:57,226 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:01,768 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:01,768 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:01,769 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:01,773 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:02,160 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:02,160 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:02,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:06,410 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:29:07,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:07,004 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:07,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:07,009 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:07,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:08,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:08,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:11,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:29:11,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:29:11,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:29:13,215 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:13,918 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:13,918 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:13,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:13,923 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:14,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:16,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:17,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:17,383 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:17,384 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:17,389 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:18,177 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:18,177 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:18,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:21,593 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:21,594 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:21,595 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:21,600 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:22,182 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:22,182 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:23,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:26,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:29:26,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:29:26,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:29:26,318 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:26,322 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:26,322 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:26,327 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:27,185 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:28,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:29,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:31,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:31,006 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:31,007 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:31,012 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:31,189 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:32,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:35,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:36,412 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:29:36,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:36,745 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:36,745 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:36,750 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:37,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:38,198 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:40,676 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:40,676 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:40,677 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:40,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:40,682 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:41,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:29:41,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:29:41,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:29:41,200 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:42,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:45,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:45,608 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:45,608 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:45,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:46,207 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:46,207 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:46,649 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:51,649 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:51,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:51,835 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:51,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:51,840 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:52,212 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:52,213 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:55,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:55,940 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:55,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:55,945 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:56,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:29:56,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:29:56,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:29:56,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:29:57,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:58,219 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:29:59,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:59,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:29:59,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:59,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:00,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:02,223 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:02,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:03,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:03,630 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:03,630 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:03,636 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:04,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:06,228 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:06,415 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:30:07,766 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:07,767 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:07,767 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:07,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:07,772 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:08,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:10,232 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:11,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:30:11,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:30:11,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:30:13,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:14,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:14,074 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:14,074 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:14,079 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:14,236 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:16,240 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:18,493 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:18,494 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:18,494 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:18,494 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:18,499 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:19,242 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:20,245 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:23,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:23,333 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:23,333 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:23,338 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:24,249 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:24,249 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:24,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:26,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:30:26,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:30:26,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:30:29,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:29,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:29,469 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:29,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:29,474 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:30,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:32,259 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:34,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:35,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:35,965 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:35,966 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:35,970 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:36,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:36,417 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:30:38,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:39,479 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:39,480 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:39,480 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:39,485 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:40,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:40,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:40,485 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:41,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:30:41,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:30:41,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:30:45,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:45,615 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:45,615 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:45,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:45,620 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:46,271 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:48,274 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:50,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:50,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:50,888 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:50,888 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:50,893 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:51,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:52,278 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:55,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:55,232 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:55,233 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:55,237 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:55,281 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:30:56,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:30:56,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:30:56,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:30:56,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:56,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:30:59,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:59,213 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:30:59,214 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:59,219 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:59,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:00,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:01,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:06,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:06,419 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:31:07,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:07,873 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:07,873 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:07,878 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:08,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:10,297 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:11,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:31:11,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:31:11,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:31:11,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:12,178 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:12,179 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:12,179 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:12,184 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:12,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:14,303 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:15,864 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:15,865 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:15,865 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:15,871 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:16,304 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:16,871 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:18,308 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:20,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:20,624 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:20,624 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:20,629 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:21,310 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:22,313 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:22,630 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:25,513 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:25,514 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:25,514 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:25,519 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:26,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:31:26,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:31:26,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:31:26,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:26,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:28,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:29,092 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:29,092 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:29,093 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:29,097 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:29,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:30,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:33,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:33,094 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:33,094 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:33,099 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:33,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:34,099 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:34,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:36,424 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:31:38,391 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:38,392 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:38,392 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:38,397 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:39,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:39,398 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:40,332 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:41,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:31:41,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:31:41,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:31:45,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:46,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:46,704 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:46,704 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:46,709 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:47,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:48,338 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:50,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:51,612 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:51,613 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:51,613 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:51,618 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:52,341 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:54,345 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:31:55,380 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:55,381 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:31:55,381 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:55,386 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:56,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:31:56,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:31:56,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:31:56,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:56,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:31:56,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:00,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:00,253 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:00,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:00,258 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:00,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:01,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:02,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:05,487 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:05,488 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:05,488 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:05,493 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:06,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:06,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:06,423 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:32:06,424 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:09,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:09,826 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:09,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:09,831 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:10,361 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:11,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:32:11,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:32:11,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:32:12,228 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:12,366 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:13,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:13,014 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:13,014 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:13,018 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:13,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:14,369 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:17,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:17,591 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:17,591 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:17,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:17,596 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:18,373 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:18,374 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:22,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:23,811 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:23,812 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:23,813 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:23,818 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:24,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:26,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:32:26,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:32:26,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:32:26,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:28,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:28,559 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:28,560 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:28,560 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:28,565 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:29,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:30,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:32,369 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:32,370 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:32,370 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:32,375 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:32,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:33,376 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:34,390 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:36,428 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:32:38,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:38,004 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:38,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:38,010 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:38,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:39,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:40,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:41,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:32:41,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:32:41,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:32:42,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:42,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:42,990 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:42,995 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:43,398 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:44,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:44,401 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:46,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:46,716 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:46,716 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:46,721 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:47,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:48,405 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:49,722 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:52,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:52,445 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:52,445 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:52,450 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:53,410 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:54,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:32:55,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:56,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:32:56,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:32:56,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:32:56,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:56,439 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:32:56,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:56,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:57,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:32:58,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:00,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:05,068 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:05,069 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:05,070 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:05,074 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:05,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:06,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:06,423 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:06,427 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:33:09,103 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:09,104 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:09,104 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:09,109 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:09,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:10,428 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:11,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:33:11,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:33:11,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:33:11,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:14,138 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:14,139 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:14,139 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:14,144 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:14,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:16,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:17,181 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:18,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:18,001 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:18,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:18,006 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:18,436 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:20,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:22,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:22,803 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:22,803 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:22,803 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:22,808 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:23,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:24,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:26,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:33:26,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:33:26,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:33:27,818 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:27,819 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:27,819 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:27,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:27,824 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:28,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:30,449 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:32,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:35,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:35,152 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:35,153 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:35,157 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:35,452 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:36,429 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:33:36,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:38,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:39,240 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:39,241 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:39,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:39,247 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:39,457 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:40,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:41,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:33:41,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:33:41,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:33:43,164 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:43,168 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:43,168 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:43,173 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:43,461 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:44,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:44,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:48,864 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:48,865 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:48,865 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:48,871 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:49,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:49,871 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:50,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:52,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:52,807 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:33:52,807 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:52,812 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:53,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:33:54,475 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:33:55,812 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:56,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:33:56,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:33:56,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:34:01,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:01,243 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:01,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:01,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:01,248 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:01,479 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:02,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:06,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:06,431 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:34:08,164 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:08,165 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:08,165 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:08,170 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:08,486 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:10,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:11,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:34:11,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:34:11,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:34:11,288 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:13,179 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:13,180 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:13,180 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:13,185 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:13,492 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:14,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:17,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:17,871 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:17,872 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:17,872 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:17,877 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:18,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:20,501 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:21,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:21,459 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:21,460 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:21,465 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:21,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:22,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:22,505 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:25,676 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:25,677 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:25,677 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:25,682 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:26,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:34:26,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:34:26,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:34:26,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:26,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:28,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:30,705 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:30,706 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:30,706 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:30,711 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:31,512 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:32,515 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:33,749 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:34,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:34,254 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:34,255 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:34,260 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:34,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:36,433 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:34:36,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:37,514 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:37,515 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:37,515 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:37,521 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:37,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:38,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:39,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:41,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:34:41,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:34:41,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:34:42,181 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:42,182 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:42,182 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:42,188 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:42,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:44,529 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:45,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:48,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:48,897 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:48,898 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:48,903 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:49,532 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:50,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:50,903 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:53,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:53,950 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:53,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:53,955 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:54,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:34:55,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:56,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:34:56,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:34:56,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:34:56,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:34:59,163 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:59,163 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:34:59,164 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:59,169 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:59,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:00,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:01,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:04,360 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:04,361 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:04,361 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:04,366 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:04,548 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:06,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:06,435 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:35:06,551 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:08,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:08,778 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:08,779 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:08,783 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:09,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:10,555 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:11,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:35:11,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:35:11,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:35:12,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:13,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:13,310 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:13,310 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:13,315 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:13,558 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:14,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:17,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:17,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:17,496 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:17,497 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:17,502 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:17,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:18,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:21,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:21,944 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:21,945 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:21,949 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:22,568 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:22,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:24,571 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:25,918 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:25,919 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:25,919 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:25,924 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:26,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:35:26,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:35:26,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:35:26,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:28,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:28,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:32,358 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:32,359 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:32,359 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:32,364 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:32,578 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:33,401 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:34,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:36,072 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:36,073 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:36,073 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:36,078 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:36,437 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:35:36,583 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:38,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:38,586 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:41,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:35:41,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:35:41,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:35:41,405 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:41,406 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:41,406 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:41,411 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:41,588 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:42,590 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:43,449 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:45,204 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:45,205 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:45,205 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:45,210 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:45,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:46,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:49,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:49,017 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:49,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:49,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:49,022 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:49,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:50,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:52,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:52,964 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:52,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:52,970 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:53,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:54,604 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:35:54,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:56,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:35:56,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:35:56,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:35:56,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:56,754 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:35:56,754 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:56,759 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:57,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:35:58,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:00,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:00,749 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:00,750 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:00,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:00,754 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:01,611 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:02,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:04,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:04,292 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:04,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:04,297 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:04,615 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:06,298 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:06,443 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:36:06,618 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:09,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:09,021 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:09,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:09,026 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:09,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:10,622 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:11,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:36:11,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:36:11,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:36:12,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:14,751 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:14,752 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:14,752 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:14,757 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:15,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:16,629 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:17,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:19,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:19,545 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:19,546 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:19,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:19,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:20,634 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:23,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:24,144 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:24,145 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:24,145 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:24,150 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:24,636 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:26,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:36:26,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:36:26,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:36:26,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:27,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:27,830 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:27,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:27,836 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:28,643 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:28,643 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:28,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:31,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:31,762 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:31,763 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:31,767 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:32,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:32,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:34,768 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:35,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:35,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:35,259 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:35,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:35,649 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:36,442 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:36:36,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:39,520 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:39,521 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:39,521 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:39,526 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:39,654 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:40,526 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:40,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:41,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:36:41,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:36:41,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:36:44,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:44,654 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:44,654 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:44,659 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:44,660 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:45,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:46,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:49,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:49,047 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:49,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:49,052 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:49,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:50,668 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:51,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:53,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:53,827 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:53,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:53,832 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:54,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:54,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:36:56,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:36:56,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:36:56,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:36:56,354 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:57,635 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:57,635 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:36:57,635 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:57,640 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:57,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:36:58,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:01,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:02,081 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:02,082 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:02,082 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:02,087 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:02,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:04,682 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:06,446 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:37:06,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:06,845 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:06,846 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:06,846 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:06,851 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:07,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:08,686 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:11,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:37:11,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:37:11,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:37:11,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:11,626 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:11,626 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:11,631 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:11,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:12,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:12,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:15,379 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:15,380 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:15,380 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:15,385 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:15,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:16,697 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:18,386 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:21,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:21,015 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:21,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:21,020 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:21,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:22,702 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:24,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:24,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:24,742 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:24,742 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:24,747 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:25,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:26,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:37:26,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:37:26,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:37:26,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:29,327 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:29,531 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:29,532 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:29,532 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:29,537 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:29,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:30,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:33,147 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:33,147 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:33,148 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:33,153 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:33,714 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:34,716 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:35,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:36,446 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:37:36,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:36,827 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:36,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:36,833 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:37,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:38,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:40,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:41,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:37:41,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:37:41,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:37:41,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:41,269 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:41,269 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:41,274 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:41,722 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:42,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:46,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:47,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:47,052 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:47,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:47,057 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:47,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:48,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:51,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:51,222 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:51,222 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:51,227 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:51,732 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:52,228 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:52,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:55,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:55,262 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:55,262 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:55,267 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:55,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:37:56,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:37:56,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:37:56,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:37:56,740 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:37:57,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:59,311 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:59,312 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:37:59,312 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:59,317 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:59,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:00,744 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:02,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:02,977 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:02,978 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:02,978 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:02,983 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:03,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:04,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:06,451 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:38:07,161 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:07,162 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:07,162 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:07,167 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:07,751 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:08,168 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:08,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:11,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:38:11,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:38:11,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:38:11,155 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:11,228 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:11,228 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:11,233 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:11,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:12,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:13,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:18,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:19,655 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:19,655 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:19,656 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:19,660 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:19,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:20,767 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:23,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:25,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:25,795 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:25,796 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:25,800 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:26,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:38:26,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:38:26,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:38:26,772 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:26,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:29,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:30,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:30,795 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:30,795 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:30,801 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:31,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:32,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:34,628 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:34,629 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:34,629 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:34,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:34,634 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:34,779 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:36,450 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:38:36,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:39,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:39,310 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:39,310 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:39,315 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:39,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:40,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:40,787 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:41,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:38:41,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:38:41,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:38:43,227 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:43,228 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:43,228 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:43,233 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:43,789 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:44,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:46,269 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:47,983 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:47,984 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:47,985 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:47,990 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:48,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:48,798 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:51,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:51,940 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:51,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:51,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:51,945 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:52,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:52,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:38:56,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:38:56,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:38:56,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:38:56,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:56,823 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:38:56,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:56,829 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:57,806 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:38:57,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:58,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:01,189 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:01,189 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:01,190 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:01,194 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:01,810 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:02,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:03,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:05,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:05,803 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:05,803 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:05,808 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:05,815 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:06,455 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:39:06,817 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:08,457 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:09,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:09,861 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:09,862 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:09,866 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:10,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:10,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:11,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:39:11,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:39:11,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:39:13,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:13,999 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:13,999 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:13,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:14,004 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:14,827 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:14,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:17,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:17,535 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:17,535 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:17,540 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:17,829 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:18,831 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:19,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:22,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:22,606 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:22,606 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:22,611 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:22,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:24,612 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:24,837 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:26,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:39:26,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:39:26,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:39:26,422 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:26,423 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:26,423 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:26,428 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:26,839 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:28,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:30,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:30,598 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:30,598 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:30,598 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:30,604 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:30,843 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:32,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:35,211 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:35,212 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:35,212 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:35,217 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:35,848 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:36,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:36,454 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:39:36,851 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:39,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:39,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:39,983 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:39,988 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:40,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:40,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:41,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:39:41,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:39:41,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:39:41,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:46,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:48,140 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:48,140 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:48,140 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:48,145 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:48,860 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:50,863 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:51,891 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:51,892 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:51,892 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:51,892 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:51,897 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:52,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:52,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:55,899 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:55,900 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:39:55,900 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:55,907 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:56,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:39:56,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:39:56,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:39:56,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:39:56,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:39:57,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:02,273 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:02,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:02,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:02,697 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:02,702 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:02,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:04,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:40:06,456 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:40:07,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:10,817 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:10,817 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:10,818 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:10,822 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:10,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:11,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:40:11,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:40:11,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:40:12,885 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:40:13,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:14,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:14,821 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:14,822 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:14,826 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:14,886 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:16,889 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:40:18,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:20,460 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:20,461 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:20,461 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:20,466 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:20,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:22,895 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:40:24,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:25,646 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:25,647 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:25,647 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:25,652 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:25,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:26,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:40:26,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:40:26,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:40:26,900 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:40:29,794 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:29,795 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:29,795 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:29,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:29,800 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:29,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:30,904 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:40:33,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:33,509 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:33,510 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:33,514 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:33,906 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:34,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:40:35,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:36,458 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:40:38,632 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:38,633 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:38,633 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:38,638 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:38,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:40,638 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:40,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:40:41,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:40:41,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:40:41,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:40:43,864 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:43,865 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:43,865 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:43,870 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:43,916 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:44,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:40:45,905 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:48,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:48,330 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:48,330 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:48,335 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:48,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:50,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:40:51,336 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:56,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:40:56,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:40:56,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:40:56,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:56,937 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:40:56,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:56,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:56,942 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:57,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:40:58,932 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:00,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:00,491 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:00,491 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:00,496 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:00,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:02,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:02,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:04,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:04,749 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:04,749 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:04,755 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:04,938 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:06,461 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:41:06,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:08,318 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:08,319 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:08,319 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:08,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:08,324 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:08,942 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:10,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:11,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:41:11,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:41:11,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:41:14,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:15,867 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:15,868 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:15,868 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:15,873 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:15,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:16,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:19,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:20,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:20,807 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:20,807 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:20,812 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:20,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:22,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:24,975 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:24,976 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:24,976 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:24,976 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:24,981 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:25,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:26,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:41:26,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:41:26,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:41:26,963 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:28,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:28,944 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:28,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:28,950 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:28,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:29,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:30,968 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:34,540 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:34,540 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:34,541 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:34,546 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:34,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:35,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:36,463 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:41:36,973 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:40,455 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:40,455 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:40,456 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:40,461 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:40,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:41,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:41:41,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:41:41,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:41:41,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:42,978 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:44,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:44,042 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:44,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:44,047 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:44,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:44,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:47,085 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:48,606 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:48,606 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:48,607 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:48,612 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:48,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:50,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:52,612 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:53,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:53,066 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:53,067 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:53,072 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:53,990 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:54,993 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:41:56,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:41:56,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:41:56,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:41:57,739 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:57,740 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:41:57,740 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:57,740 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:57,745 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:57,995 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:41:58,997 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:02,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:02,154 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:02,155 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:02,159 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:03,002 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:03,002 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:03,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:06,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:06,024 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:06,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:06,029 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:06,465 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:42:07,006 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:07,006 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:08,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:11,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:42:11,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:42:11,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:42:11,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:11,731 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:11,731 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:11,736 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:12,010 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:13,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:13,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:18,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:20,282 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:20,283 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:20,283 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:20,289 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:21,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:21,020 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:24,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:24,446 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:24,447 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:24,447 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:24,452 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:25,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:26,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:42:26,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:42:26,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:42:27,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:28,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:28,336 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:28,336 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:28,340 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:29,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:29,377 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:31,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:32,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:32,134 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:32,134 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:32,139 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:33,034 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:33,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:35,140 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:36,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:36,150 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:36,150 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:36,155 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:36,470 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:42:37,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:37,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:40,472 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:41,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:42:41,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:42:41,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:42:44,713 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:44,714 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:44,714 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:44,719 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:45,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:45,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:47,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:48,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:48,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:48,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:48,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:49,047 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:50,883 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:51,051 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:54,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:54,294 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:54,294 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:54,300 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:55,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:55,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:42:56,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:42:56,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:42:56,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:42:56,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:58,269 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:58,269 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:42:58,270 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:58,274 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:59,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:42:59,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:01,313 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:04,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:04,298 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:04,298 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:04,303 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:05,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:05,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:06,469 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:43:06,471 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:09,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:09,301 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:09,301 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:09,306 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:10,069 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:11,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:43:11,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:43:11,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:43:11,071 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:12,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:14,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:14,868 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:14,868 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:14,873 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:15,075 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:17,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:17,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:18,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:18,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:18,278 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:18,283 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:19,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:19,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:22,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:22,107 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:22,107 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:22,112 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:23,088 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:23,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:23,112 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:25,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:25,911 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:25,911 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:25,916 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:26,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:43:26,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:43:26,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:43:26,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:27,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:28,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:29,785 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:29,785 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:29,786 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:29,790 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:30,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:31,101 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:33,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:33,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:33,860 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:33,860 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:33,865 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:34,103 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:35,106 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:36,474 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:43:38,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:38,067 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:38,067 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:38,072 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:38,108 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:39,072 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:39,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:41,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:43:41,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:43:41,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:43:41,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:41,674 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:41,674 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:41,679 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:42,113 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:43,117 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:44,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:47,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:47,582 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:47,582 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:47,587 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:48,122 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:49,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:50,588 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:51,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:51,323 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:51,323 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:51,328 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:52,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:53,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:43:56,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:43:56,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:43:56,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:43:56,298 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:56,977 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:56,978 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:43:56,978 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:56,983 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:57,136 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:43:59,141 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:01,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:01,576 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:01,576 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:01,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:01,581 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:02,142 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:03,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:05,423 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:05,423 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:05,424 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:05,429 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:06,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:06,473 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:44:07,152 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:07,475 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:11,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:44:11,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:44:11,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:44:13,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:13,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:13,828 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:13,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:13,833 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:14,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:15,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:17,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:17,997 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:17,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:18,003 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:18,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:19,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:19,169 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:24,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:26,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:44:26,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:44:26,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:44:26,425 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:26,426 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:26,426 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:26,430 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:27,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:29,183 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:29,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:30,692 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:30,693 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:30,693 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:30,699 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:31,184 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:33,189 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:34,699 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:34,900 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:34,901 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:34,902 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:34,907 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:35,190 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:36,476 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:44:37,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:39,891 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:39,895 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:39,895 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:39,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:39,900 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:40,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:41,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:44:41,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:44:41,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:44:41,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:43,399 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:43,400 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:43,400 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:43,405 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:44,203 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:45,207 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:45,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:47,393 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:47,394 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:47,394 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:47,399 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:48,208 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:49,213 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:51,400 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:51,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:51,958 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:51,958 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:51,963 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:52,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:53,218 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:56,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:44:56,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:44:56,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:44:56,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:56,312 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:44:56,313 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:56,317 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:57,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:44:57,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:44:57,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:00,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:00,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:00,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:00,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:01,228 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:02,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:03,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:45:04,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:04,996 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:04,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:05,002 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:05,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:06,478 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:45:07,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:45:08,479 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:11,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:45:11,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:45:11,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:45:13,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:13,962 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:13,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:13,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:13,967 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:14,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:15,249 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:45:18,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:18,016 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:18,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:18,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:18,252 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:19,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:19,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:45:24,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:24,090 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:24,091 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:24,091 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:24,096 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:24,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:25,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:45:26,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:45:26,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:45:26,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:45:28,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:28,883 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:28,883 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:28,888 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:29,269 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:29,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:31,274 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:45:34,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:36,480 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:45:38,077 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:38,078 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:38,078 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:38,083 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:38,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:39,285 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:45:40,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:41,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:45:41,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:45:41,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:45:42,086 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:42,087 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:42,087 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:42,092 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:42,287 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:43,290 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:45:45,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:47,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:47,623 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:47,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:47,628 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:48,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:49,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:45:50,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:51,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:51,991 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:51,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:51,996 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:52,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:53,304 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:45:55,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:55,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:45:55,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:55,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:55,937 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:56,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:45:56,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:45:56,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:45:56,307 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:45:57,311 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:01,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:01,817 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:01,818 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:01,818 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:01,823 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:02,315 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:03,320 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:06,482 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:46:06,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:06,772 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:06,773 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:06,773 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:06,778 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:07,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:09,328 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:11,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:46:11,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:46:11,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:46:11,456 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:11,457 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:11,457 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:11,462 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:11,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:12,331 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:13,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:16,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:16,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:16,883 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:16,883 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:16,888 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:17,338 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:19,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:20,594 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:20,595 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:20,595 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:20,601 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:21,344 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:21,601 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:23,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:25,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:25,893 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:25,894 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:25,899 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:26,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:46:26,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:46:26,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:46:26,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:27,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:27,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:31,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:31,554 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:31,555 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:31,559 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:32,359 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:32,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:33,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:35,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:35,332 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:35,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:35,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:35,365 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:36,484 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:46:37,369 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:38,485 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:39,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:39,974 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:39,974 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:39,980 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:40,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:41,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:46:41,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:46:41,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:46:41,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:43,732 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:43,733 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:43,733 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:43,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:43,738 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:44,376 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:45,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:48,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:49,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:49,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:49,258 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:49,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:49,384 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:51,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:54,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:55,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:55,347 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:46:55,347 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:55,352 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:55,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:46:56,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:46:56,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:46:56,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:46:57,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:46:59,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:03,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:03,821 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:03,821 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:03,826 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:04,405 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:04,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:05,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:06,489 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:47:07,778 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:07,779 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:07,779 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:07,783 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:08,411 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:09,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:10,784 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:11,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:47:11,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:47:11,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:47:12,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:12,373 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:12,373 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:12,378 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:12,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:13,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:16,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:16,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:16,555 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:16,555 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:16,560 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:17,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:17,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:20,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:20,375 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:20,375 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:20,380 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:20,429 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:21,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:22,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:23,800 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:23,801 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:23,801 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:23,806 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:24,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:25,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:26,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:47:26,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:47:26,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:47:27,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:27,810 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:27,810 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:27,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:27,815 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:28,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:29,445 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:31,884 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:31,885 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:31,885 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:31,890 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:32,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:32,891 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:33,450 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:36,488 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:47:36,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:36,554 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:36,554 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:36,559 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:37,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:37,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:38,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:41,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:47:41,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:47:41,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:47:44,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:45,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:45,018 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:45,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:45,023 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:45,465 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:47,469 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:49,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:49,207 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:49,207 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:49,212 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:49,471 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:50,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:51,475 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:52,763 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:52,763 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:52,764 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:52,769 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:53,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:47:55,482 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:47:55,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:56,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:47:56,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:47:56,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:47:58,864 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:58,865 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:47:58,865 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:58,870 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:59,486 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:00,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:01,491 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:05,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:06,493 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:48:07,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:07,095 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:07,095 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:07,100 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:07,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:09,501 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:11,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:48:11,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:48:11,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:48:11,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:12,608 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:12,608 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:12,608 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:12,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:13,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:13,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:16,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:16,271 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:16,271 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:16,276 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:16,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:17,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:17,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:20,567 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:20,568 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:20,568 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:20,573 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:21,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:21,520 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:22,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:24,760 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:24,761 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:24,761 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:24,766 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:25,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:26,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:48:26,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:48:26,023 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:48:27,528 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:28,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:28,523 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:28,523 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:28,523 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:28,530 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:28,531 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:29,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:32,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:32,929 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:32,930 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:32,935 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:33,536 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:33,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:35,540 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:36,492 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:48:39,494 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:41,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:48:41,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:48:41,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:48:41,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:41,288 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:41,288 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:41,293 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:41,547 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:43,552 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:45,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:45,412 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:45,413 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:45,413 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:45,418 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:45,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:47,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:49,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:49,585 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:49,585 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:49,590 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:50,559 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:50,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:51,563 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:53,425 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:53,426 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:48:53,426 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:53,434 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:53,564 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:48:55,567 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:48:56,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:48:56,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:48:56,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:48:56,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:00,632 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:00,633 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:00,633 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:00,638 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:01,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:01,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:01,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:05,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:05,285 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:05,285 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:05,290 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:05,575 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:06,498 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:49:07,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:07,578 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:09,360 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:09,361 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:09,361 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:09,365 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:09,579 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:11,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:49:11,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:49:11,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:49:11,582 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:13,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:13,550 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:13,551 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:13,551 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:13,556 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:13,585 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:15,588 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:17,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:17,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:17,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:17,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:18,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:18,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:19,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:22,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:22,699 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:22,699 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:22,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:23,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:23,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:24,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:26,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:49:26,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:49:26,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:49:28,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:28,214 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:28,214 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:28,219 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:28,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:29,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:30,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:32,144 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:32,145 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:32,145 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:32,150 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:32,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:33,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:36,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:36,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:36,159 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:36,160 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:36,164 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:36,497 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:49:36,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:37,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:40,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:40,696 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:40,696 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:40,700 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:41,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:49:41,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:49:41,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:49:41,415 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:41,616 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:41,616 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:44,691 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:44,692 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:44,692 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:44,697 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:45,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:45,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:46,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:49,588 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:49,589 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:49,589 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:49,594 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:49,622 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:51,625 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:52,595 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:53,912 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:53,913 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:53,913 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:53,918 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:54,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:49:55,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:49:56,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:49:56,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:49:56,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:49:58,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:58,975 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:58,976 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:49:58,976 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:58,981 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:59,634 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:01,636 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:03,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:03,017 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:03,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:03,022 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:03,638 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:04,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:05,641 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:06,502 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:50:08,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:08,679 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:08,679 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:08,684 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:09,645 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:09,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:09,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:11,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:50:11,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:50:11,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:50:12,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:12,263 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:12,264 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:12,269 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:12,649 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:13,651 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:15,305 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:16,705 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:16,705 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:16,706 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:16,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:17,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:17,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:20,711 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:24,362 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:24,363 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:24,363 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:24,368 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:24,661 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:25,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:26,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:50:26,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:50:26,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:50:26,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:29,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:29,131 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:29,131 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:29,135 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:29,666 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:31,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:32,173 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:33,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:33,266 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:33,266 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:33,271 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:33,670 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:35,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:36,501 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:50:37,502 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:37,878 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:37,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:37,879 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:37,884 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:38,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:39,678 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:41,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:50:41,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:50:41,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:50:42,664 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:42,664 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:42,665 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:42,665 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:42,669 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:42,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:43,682 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:46,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:46,998 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:46,999 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:47,003 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:47,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:48,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:49,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:51,787 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:51,788 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:51,788 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:51,793 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:52,689 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:53,691 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:50:53,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:56,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:50:56,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:50:56,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:50:56,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:56,994 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:50:56,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:56,999 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:57,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:50:59,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:59,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:01,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:01,517 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:01,517 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:01,523 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:01,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:03,702 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:04,523 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:05,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:05,965 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:05,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:05,970 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:06,503 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:51:06,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:07,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:10,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:10,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:10,649 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:10,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:10,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:10,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:11,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:51:11,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:51:11,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:51:11,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:14,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:14,019 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:14,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:14,024 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:14,715 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:15,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:16,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:18,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:18,035 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:18,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:18,040 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:18,719 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:19,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:22,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:22,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:22,435 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:22,435 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:22,440 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:22,724 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:23,727 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:26,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:51:26,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:51:26,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:51:26,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:26,238 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:26,238 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:26,243 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:26,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:27,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:27,732 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:29,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:29,946 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:29,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:29,951 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:30,733 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:31,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:32,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:36,505 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:51:36,785 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:36,786 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:36,786 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:36,791 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:37,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:37,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:38,791 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:40,786 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:40,787 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:40,787 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:40,792 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:41,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:51:41,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:51:41,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:51:41,745 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:41,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:44,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:45,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:45,364 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:45,364 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:45,369 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:45,748 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:47,751 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:49,405 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:49,875 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:49,876 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:49,876 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:49,881 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:50,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:51,755 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:54,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:54,679 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:51:54,679 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:54,679 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:54,684 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:54,758 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:51:55,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:51:56,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:51:56,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:51:56,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:52:00,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:03,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:03,331 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:03,331 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:03,336 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:03,765 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:05,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:05,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:06,507 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:52:06,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:06,749 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:06,749 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:06,754 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:06,769 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:07,771 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:10,754 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:11,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:52:11,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:52:11,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:52:12,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:12,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:12,789 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:12,794 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:13,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:13,779 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:15,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:18,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:18,330 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:18,330 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:18,335 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:18,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:19,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:21,336 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:26,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:52:26,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:52:26,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:52:26,356 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:26,356 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:26,357 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:26,357 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:26,362 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:26,789 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:27,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:29,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:29,654 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:29,654 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:29,659 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:29,793 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:31,659 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:31,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:33,301 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:33,302 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:33,302 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:33,308 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:33,798 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:35,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:36,512 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:52:37,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:37,026 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:37,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:37,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:37,031 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:37,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:39,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:40,857 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:40,858 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:40,858 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:40,863 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:41,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:52:41,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:52:41,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:52:41,808 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:41,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:42,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:45,603 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:45,604 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:45,604 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:45,609 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:45,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:47,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:47,814 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:52,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:53,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:53,791 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:53,791 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:53,796 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:53,817 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:55,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:52:56,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:52:56,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:52:56,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:52:58,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:58,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:58,302 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:52:58,302 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:58,307 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:58,824 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:52:59,826 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:02,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:02,532 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:02,533 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:02,538 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:02,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:03,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:03,830 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:06,511 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:53:06,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:06,944 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:06,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:06,949 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:07,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:07,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:08,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:10,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:10,862 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:10,862 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:10,867 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:11,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:53:11,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:53:11,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:53:11,839 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:11,839 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:14,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:16,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:16,331 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:16,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:16,336 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:16,843 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:17,845 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:19,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:20,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:20,022 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:20,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:20,027 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:20,848 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:21,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:24,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:24,020 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:24,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:24,025 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:24,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:25,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:25,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:26,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:53:26,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:53:26,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:53:29,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:29,252 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:29,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:29,257 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:29,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:30,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:31,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:35,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:36,517 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:53:36,573 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:36,574 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:36,574 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:36,579 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:36,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:37,866 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:40,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:40,243 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:40,243 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:40,247 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:40,868 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:41,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:53:41,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:53:41,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:53:41,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:41,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:44,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:44,502 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:44,503 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:44,507 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:44,872 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:45,874 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:46,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:51,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:51,020 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:51,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:51,025 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:51,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:51,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:53:52,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:55,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:55,603 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:53:55,603 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:55,609 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:55,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:53:56,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:53:56,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:53:56,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:53:57,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:57,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:00,396 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:00,397 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:00,397 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:00,402 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:00,889 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:01,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:02,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:04,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:04,340 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:04,340 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:04,345 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:04,893 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:05,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:06,516 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:54:07,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:11,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:54:11,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:54:11,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:54:12,787 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:12,787 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:12,788 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:12,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:12,792 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:12,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:13,904 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:17,342 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:17,343 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:17,343 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:17,348 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:17,906 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:18,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:19,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:21,165 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:21,166 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:21,167 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:21,171 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:21,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:23,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:24,172 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:26,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:54:26,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:54:26,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:54:26,219 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:26,273 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:26,273 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:26,278 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:26,916 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:27,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:29,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:30,423 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:30,424 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:30,424 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:30,429 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:30,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:31,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:34,113 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:34,113 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:34,114 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:34,119 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:34,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:35,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:35,928 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:36,521 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:54:37,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:37,926 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:37,926 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:37,931 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:37,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:39,934 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:40,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:41,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:54:41,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:54:41,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:54:42,804 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:42,805 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:42,805 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:42,810 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:42,936 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:43,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:46,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:46,982 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:46,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:46,983 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:46,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:47,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:47,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:51,126 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:51,127 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:51,127 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:51,132 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:51,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:51,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:52,132 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:55,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:55,897 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:55,897 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:55,902 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:55,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:54:56,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:54:56,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:54:56,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:54:57,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:57,953 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:54:59,424 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:59,425 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:54:59,425 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:59,430 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:59,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:01,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:02,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:05,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:05,335 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:05,336 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:05,341 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:05,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:06,520 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:55:07,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:07,963 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:11,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:55:11,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:55:11,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:55:12,871 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:12,871 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:12,872 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:12,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:12,877 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:12,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:13,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:17,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:17,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:17,522 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:17,527 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:17,972 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:18,527 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:19,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:22,125 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:22,126 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:22,126 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:22,131 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:22,977 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:23,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:24,132 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:26,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:55:26,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:55:26,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:55:26,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:26,317 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:26,317 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:26,322 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:26,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:27,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:29,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:31,505 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:31,506 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:31,506 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:31,511 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:31,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:33,990 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:34,512 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:36,118 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:36,118 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:36,119 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:36,123 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:36,526 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:55:36,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:37,994 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:39,528 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:41,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:55:41,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:55:41,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:55:42,099 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:42,099 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:42,100 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:42,104 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:42,997 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:44,000 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:45,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:46,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:46,805 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:46,806 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:46,810 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:47,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:48,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:50,811 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:51,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:51,867 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:51,867 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:51,872 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:52,006 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:55:54,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:55:55,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:56,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:55:56,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:55:56,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:55:58,763 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:58,764 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:55:58,764 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:58,769 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:59,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:00,016 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:01,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:02,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:02,873 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:02,873 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:02,878 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:03,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:04,023 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:06,525 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:56:06,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:06,745 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:06,745 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:06,750 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:07,025 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:07,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:08,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:10,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:10,810 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:10,810 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:10,815 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:11,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:56:11,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:56:11,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:56:11,030 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:12,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:13,305 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:14,891 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:14,891 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:14,892 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:14,896 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:15,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:16,038 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:18,640 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:18,641 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:18,641 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:18,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:18,646 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:19,040 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:20,042 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:22,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:22,830 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:22,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:22,835 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:23,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:23,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:24,047 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:26,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:56:26,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:56:26,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:56:29,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:29,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:29,323 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:29,323 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:29,328 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:30,051 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:32,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:33,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:33,168 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:33,168 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:33,173 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:34,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:34,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:35,173 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:36,527 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:56:37,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:37,404 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:37,404 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:37,409 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:38,059 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:40,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:40,409 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:41,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:56:41,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:56:41,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:56:41,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:41,969 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:41,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:41,974 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:42,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:44,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:45,639 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:45,640 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:45,640 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:45,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:45,645 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:46,068 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:48,071 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:50,646 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:53,128 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:53,129 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:53,129 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:53,134 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:54,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:56:54,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:56:56,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:56:56,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:56:56,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:56:56,333 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:57,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:57,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:56:57,789 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:57,793 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:58,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:00,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:01,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:06,529 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:57:06,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:06,774 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:06,774 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:06,778 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:07,087 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:07,779 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:08,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:11,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:57:11,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:57:11,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:57:11,523 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:11,524 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:11,524 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:11,529 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:12,092 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:13,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:14,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:16,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:16,303 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:16,304 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:16,308 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:17,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:18,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:19,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:20,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:20,057 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:20,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:20,062 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:20,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:22,105 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:24,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:24,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:24,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:24,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:24,473 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:25,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:26,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:57:26,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:57:26,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:57:26,109 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:29,940 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:29,941 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:29,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:29,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:29,946 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:30,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:32,115 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:34,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:34,035 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:34,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:34,040 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:34,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:35,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:36,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:36,531 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:57:38,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:38,065 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:38,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:38,070 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:38,120 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:40,070 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:40,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:41,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:57:41,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:57:41,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:57:41,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:41,699 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:41,699 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:41,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:42,125 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:44,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:45,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:50,232 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:50,233 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:50,233 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:50,238 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:51,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:51,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:52,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:54,709 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:54,710 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:54,710 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:54,715 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:55,138 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:57:56,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:57:56,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:57:56,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:57:56,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:57:56,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:58,456 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:58,457 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:57:58,457 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:58,462 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:59,142 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:00,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:01,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:03,155 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:03,155 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:03,156 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:03,160 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:04,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:04,149 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:06,536 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:58:06,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:08,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:08,263 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:08,263 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:08,268 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:09,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:10,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:11,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:58:11,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:58:11,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:58:12,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:12,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:12,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:12,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:12,472 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:13,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:14,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:16,211 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:16,211 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:16,212 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:16,216 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:17,161 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:18,164 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:18,217 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:19,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:19,997 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:19,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:20,002 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:20,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:22,169 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:24,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:24,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:25,000 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:25,000 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:25,005 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:25,171 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:26,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:58:26,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:58:26,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:58:26,173 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:28,638 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:28,639 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:28,639 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:28,644 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:29,175 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:29,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:30,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:32,440 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:32,440 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:32,441 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:32,446 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:33,180 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:34,182 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:35,446 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:36,535 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:58:40,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:40,004 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:40,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:40,009 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:40,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:41,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:41,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:58:41,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:58:41,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:58:42,189 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:44,144 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:44,145 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:44,145 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:44,150 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:44,190 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:46,189 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:46,193 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:48,700 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:48,701 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:48,701 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:48,706 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:49,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:50,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:51,707 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:55,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:55,020 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:55,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:55,025 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:55,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:58:56,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:58:56,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:58:56,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:58:56,204 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:58:57,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:58,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:58,967 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:58:58,967 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:58,972 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:59,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:00,208 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:03,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:06,540 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:59:07,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:07,445 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:07,445 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:07,450 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:08,215 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:08,215 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:08,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:11,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:59:11,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:59:11,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:59:11,583 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:11,584 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:11,584 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:11,589 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:12,217 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:13,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:14,222 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:16,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:16,590 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:16,590 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:16,595 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:17,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:18,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:19,595 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:20,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:20,888 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:20,888 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:20,893 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:21,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:22,232 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:24,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:25,533 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:25,534 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:25,534 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:25,539 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:26,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:59:26,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:59:26,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:59:26,234 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:28,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:30,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:30,976 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:30,976 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:30,977 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:30,981 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:31,240 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:32,242 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:34,919 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:34,920 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:34,920 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:34,925 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:35,244 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:35,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:36,247 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:36,539 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 15:59:38,656 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:38,657 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:38,657 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:38,662 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:39,248 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:40,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:41,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:59:41,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:59:41,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:59:41,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:42,842 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:42,843 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:42,843 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:42,848 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:43,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:44,255 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:46,885 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:47,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:47,509 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:47,509 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:47,514 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:48,258 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:50,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:52,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:52,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:52,880 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:52,881 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:52,885 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:53,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:54,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 15:59:56,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:59:56,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:59:56,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:59:57,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:57,028 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 15:59:57,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:57,033 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:57,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 15:59:58,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:58,271 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:02,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:02,094 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:02,095 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:02,099 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:02,273 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:03,100 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:04,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:06,246 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:06,247 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:06,247 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:06,252 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:06,277 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:06,544 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:00:08,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:08,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:11,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:00:11,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:00:11,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:00:14,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:14,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:14,404 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:14,404 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:14,409 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:15,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:16,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:18,299 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:18,300 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:18,300 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:18,305 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:19,290 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:19,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:20,293 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:22,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:22,253 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:22,254 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:22,259 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:22,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:24,297 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:25,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:26,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:00:26,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:00:26,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:00:26,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:26,936 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:26,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:26,941 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:27,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:28,303 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:30,977 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:31,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:31,333 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:31,333 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:31,338 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:32,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:32,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:36,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:36,543 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:00:39,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:39,583 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:39,583 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:39,588 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:40,310 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:41,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:00:41,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:00:41,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:00:42,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:42,313 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:44,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:44,142 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:44,142 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:44,147 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:44,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:46,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:47,843 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:47,844 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:47,844 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:47,844 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:47,849 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:48,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:50,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:51,987 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:51,988 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:51,989 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:51,993 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:52,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:52,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:54,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:00:55,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:55,658 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:00:55,658 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:55,662 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:56,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:00:56,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:00:56,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:00:56,328 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:00:58,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:58,332 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:02,755 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:02,755 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:02,755 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:02,760 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:03,335 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:03,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:04,337 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:06,546 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:01:08,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:08,743 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:08,743 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:08,748 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:09,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:09,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:10,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:11,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:01:11,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:01:11,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:01:12,799 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:12,800 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:12,800 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:12,805 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:13,345 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:14,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:14,842 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:16,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:16,971 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:16,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:16,976 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:17,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:18,352 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:19,977 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:22,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:22,269 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:22,269 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:22,274 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:22,355 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:24,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:25,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:26,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:01:26,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:01:26,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:01:26,485 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:26,485 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:26,486 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:26,491 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:27,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:28,363 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:30,530 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:31,572 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:31,573 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:31,573 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:31,578 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:32,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:32,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:35,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:36,548 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:01:36,675 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:36,676 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:36,676 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:36,681 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:37,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:38,372 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:40,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:41,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:01:41,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:01:41,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:01:43,113 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:43,113 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:43,114 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:43,118 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:43,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:44,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:46,157 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:46,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:46,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:46,695 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:46,700 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:47,380 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:48,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:50,476 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:50,476 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:50,477 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:50,482 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:51,384 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:51,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:52,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:54,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:54,277 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:54,277 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:54,282 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:54,388 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:01:56,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:01:56,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:01:56,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:01:56,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:01:57,342 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:59,095 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:59,096 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:01:59,096 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:59,102 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:59,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:00,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:03,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:03,010 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:03,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:03,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:03,014 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:03,398 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:04,401 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:06,550 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:02:08,551 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:08,661 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:08,662 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:08,662 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:08,667 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:09,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:10,406 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:11,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:02:11,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:02:11,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:02:12,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:12,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:12,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:12,937 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:13,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:13,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:14,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:16,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:16,737 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:16,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:16,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:17,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:18,416 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:19,743 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:21,155 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:21,156 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:21,156 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:21,161 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:21,419 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:22,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:25,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:26,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:02:26,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:02:26,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:02:26,146 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:26,321 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:26,321 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:26,326 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:26,423 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:28,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:29,731 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:29,732 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:29,732 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:29,737 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:30,428 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:30,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:32,431 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:33,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:33,733 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:33,734 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:33,739 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:34,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:35,739 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:36,436 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:36,552 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:02:38,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:38,546 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:38,547 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:38,552 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:39,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:40,440 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:41,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:02:41,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:02:41,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:02:41,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:46,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:46,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:46,894 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:46,894 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:46,899 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:47,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:48,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:50,426 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:50,427 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:50,427 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:50,432 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:50,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:51,432 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:52,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:54,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:54,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:54,258 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:54,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:54,452 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:02:56,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:02:56,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:02:56,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:02:56,457 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:02:57,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:58,695 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:58,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:02:58,696 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:58,701 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:59,458 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:00,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:02,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:02,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:02,962 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:02,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:02,967 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:03,461 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:04,463 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:06,554 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:03:07,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:07,554 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:07,554 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:07,559 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:08,468 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:08,468 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:08,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:11,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:03:11,899 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:03:11,899 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:03:12,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:12,875 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:12,875 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:12,880 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:13,472 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:13,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:14,475 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:17,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:17,655 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:17,655 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:17,660 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:18,479 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:18,479 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:19,661 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:24,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:24,062 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:24,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:24,067 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:24,483 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:25,067 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:26,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:03:26,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:03:26,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:03:26,486 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:27,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:27,823 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:27,823 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:27,828 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:28,487 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:30,490 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:30,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:32,336 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:32,337 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:32,337 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:32,342 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:32,492 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:34,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:36,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:36,214 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:36,214 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:36,214 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:36,219 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:36,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:36,559 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:03:38,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:40,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:40,340 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:40,340 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:40,345 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:40,500 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:41,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:03:41,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:03:41,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:03:41,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:42,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:44,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:44,862 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:44,862 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:44,867 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:45,505 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:46,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:46,906 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:48,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:48,690 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:48,691 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:48,696 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:49,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:50,512 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:52,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:52,473 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:52,474 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:52,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:52,479 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:52,514 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:03:54,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:03:56,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:03:56,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:03:56,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:03:58,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:58,379 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:58,380 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:03:58,380 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:58,385 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:58,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:00,524 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:02,756 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:02,757 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:02,757 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:02,762 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:03,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:03,763 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:04,528 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:06,558 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:04:07,219 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:07,219 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:07,220 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:07,225 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:07,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:08,533 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:09,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:11,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:04:11,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:04:11,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:04:11,071 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:11,328 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:11,328 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:11,333 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:11,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:12,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:14,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:16,792 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:16,792 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:16,793 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:16,798 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:17,540 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:18,542 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:19,799 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:22,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:22,922 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:22,922 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:22,928 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:23,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:24,548 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:24,928 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:26,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:04:26,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:04:26,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:04:27,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:27,034 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:27,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:27,040 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:27,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:28,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:30,077 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:35,077 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:35,364 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:35,365 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:35,365 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:35,370 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:35,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:36,560 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:36,563 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:04:39,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:39,390 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:39,391 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:39,396 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:39,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:40,396 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:40,564 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:41,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:04:41,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:04:41,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:04:43,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:43,214 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:43,214 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:43,219 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:43,566 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:44,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:46,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:49,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:49,751 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:49,751 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:49,756 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:50,575 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:50,575 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:51,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:54,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:55,000 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:04:55,000 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:55,005 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:55,579 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:04:56,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:04:56,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:04:56,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:04:56,582 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:04:57,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:00,511 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:00,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:00,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:00,516 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:00,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:02,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:02,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:05,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:05,070 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:05,070 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:05,075 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:05,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:06,562 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:05:06,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:07,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:08,920 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:08,921 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:08,921 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:08,927 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:09,593 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:10,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:11,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:05:11,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:05:11,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:05:12,727 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:12,727 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:12,728 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:12,728 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:12,733 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:13,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:14,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:17,684 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:17,685 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:17,685 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:17,690 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:18,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:18,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:18,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:23,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:23,090 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:23,090 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:23,096 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:23,610 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:24,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:24,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:26,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:05:26,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:05:26,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:05:27,402 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:27,403 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:27,403 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:27,408 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:27,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:28,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:29,445 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:30,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:30,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:30,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:30,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:31,619 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:32,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:34,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:35,665 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:35,666 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:35,666 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:35,671 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:36,567 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:05:36,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:36,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:39,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:39,859 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:39,859 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:39,864 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:40,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:40,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:40,864 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:41,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:05:41,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:05:41,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:05:43,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:43,678 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:43,678 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:43,683 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:44,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:44,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:46,721 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:47,706 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:47,706 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:47,707 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:47,711 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:48,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:48,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:51,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:51,705 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:51,705 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:51,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:52,643 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:52,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:52,710 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:55,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:55,268 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:55,268 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:55,273 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:55,647 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:05:56,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:05:56,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:05:56,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:05:56,649 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:05:58,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:58,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:58,925 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:05:58,926 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:58,931 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:59,651 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:00,654 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:03,532 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:03,533 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:03,533 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:03,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:03,539 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:03,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:04,658 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:06,566 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:06:08,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:08,248 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:08,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:08,253 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:08,661 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:09,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:10,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:11,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:06:11,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:06:11,030 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:06:12,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:12,650 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:12,651 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:12,656 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:12,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:14,670 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:14,693 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:16,222 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:16,223 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:16,223 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:16,228 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:16,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:18,674 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:20,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:20,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:20,281 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:20,282 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:20,287 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:20,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:22,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:23,631 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:23,632 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:23,632 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:23,637 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:23,680 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:24,682 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:25,638 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:26,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:06:26,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:06:26,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:06:30,400 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:30,401 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:30,401 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:30,406 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:30,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:31,445 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:32,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:34,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:34,017 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:34,017 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:34,022 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:34,691 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:36,571 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:06:36,573 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:36,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:38,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:38,649 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:38,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:38,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:38,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:40,696 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:41,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:06:41,030 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:06:41,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:06:42,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:42,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:42,545 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:42,545 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:42,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:42,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:44,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:46,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:46,332 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:46,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:46,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:46,702 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:47,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:48,705 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:51,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:51,235 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:51,236 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:51,240 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:51,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:52,710 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:53,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:55,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:55,190 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:55,191 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:55,195 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:55,713 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:06:56,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:06:56,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:06:56,030 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:06:56,716 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:06:58,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:59,635 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:59,635 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:06:59,636 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:59,641 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:59,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:00,720 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:03,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:03,383 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:03,383 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:03,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:03,388 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:03,722 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:04,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:06,571 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:07:07,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:07,349 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:07,349 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:07,354 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:07,726 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:08,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:09,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:11,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:07:11,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:07:11,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:07:11,111 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:11,337 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:11,338 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:11,343 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:11,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:12,733 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:15,343 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:20,344 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:24,104 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:24,104 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:24,105 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:24,110 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:24,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:26,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:07:26,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:07:26,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:07:26,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:26,745 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:28,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:28,793 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:28,794 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:28,799 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:29,748 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:30,751 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:31,837 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:32,778 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:32,779 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:32,779 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:32,784 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:33,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:34,755 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:36,577 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:07:36,918 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:36,919 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:36,919 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:36,919 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:36,924 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:37,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:38,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:40,713 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:40,714 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:40,714 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:40,719 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:40,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:41,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:07:41,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:07:41,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:07:42,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:42,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:45,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:45,485 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:45,485 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:45,490 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:45,766 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:46,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:47,529 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:49,324 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:49,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:49,325 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:49,330 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:49,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:50,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:53,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:53,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:53,383 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:53,384 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:53,388 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:53,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:54,777 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:07:56,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:07:56,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:07:56,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:07:58,105 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:58,105 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:07:58,105 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:58,110 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:58,781 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:07:59,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:00,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:02,487 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:02,487 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:08:02,488 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:02,492 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:02,786 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:08:04,493 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:04,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:06,575 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:08:07,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:07,255 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:08:07,255 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:07,260 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:07,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:08:08,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:10,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:11,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:08:11,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:08:11,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:08:15,646 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:15,646 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:08:15,647 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:15,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:15,651 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:15,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:08:16,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:20,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:20,219 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:08:20,219 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:20,224 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:20,804 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:08:21,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:22,807 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:25,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:25,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:08:25,462 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:25,467 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:25,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:08:26,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:08:26,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:08:26,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:08:26,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:26,812 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:29,486 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:29,486 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:08:29,486 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:29,492 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:29,814 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:08:30,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:31,529 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:36,416 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:36,416 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:08:36,417 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:36,422 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:36,577 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:08:36,578 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:36,820 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:08:38,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:40,598 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:40,598 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:08:40,599 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:40,604 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:40,824 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:08:41,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:08:41,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:08:41,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:08:42,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:42,827 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:47,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:47,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:47,288 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:08:47,288 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:47,293 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:47,830 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:08:48,833 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:51,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:51,236 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:08:51,236 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:51,241 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:51,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:08:52,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:52,837 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:08:56,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:08:56,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:08:56,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:08:57,318 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:00,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:00,331 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:00,331 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:00,335 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:00,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:02,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:02,845 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:05,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:05,032 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:05,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:05,037 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:05,847 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:06,579 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:09:06,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:07,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:09,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:09,167 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:09,167 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:09,172 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:09,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:10,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:11,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:09:11,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:09:11,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:09:13,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:13,687 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:13,688 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:13,688 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:13,693 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:13,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:14,859 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:18,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:18,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:18,768 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:18,768 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:18,773 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:18,862 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:20,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:23,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:23,353 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:23,354 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:23,358 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:23,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:24,359 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:24,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:26,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:09:26,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:09:26,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:09:26,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:26,922 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:26,923 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:26,927 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:27,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:28,874 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:29,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:31,424 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:31,425 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:31,425 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:31,430 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:31,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:32,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:35,431 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:36,581 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:09:37,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:37,066 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:37,067 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:37,071 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:37,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:38,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:41,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:09:41,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:09:41,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:09:41,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:43,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:43,203 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:43,203 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:43,207 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:43,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:44,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:47,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:50,118 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:50,119 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:50,119 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:50,124 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:50,895 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:50,895 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:53,124 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:56,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:56,019 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:56,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:56,024 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:56,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:09:56,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:09:56,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:09:56,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:09:56,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:09:58,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:59,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:59,944 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:09:59,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:59,949 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:00,906 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:00,906 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:03,801 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:03,802 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:03,802 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:03,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:03,808 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:03,907 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:04,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:06,586 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:10:07,783 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:07,783 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:07,784 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:07,788 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:07,912 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:08,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:09,789 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:11,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:10:11,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:10:11,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:10:13,472 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:13,473 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:13,473 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:13,478 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:13,918 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:14,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:15,514 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:17,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:17,835 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:17,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:17,840 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:17,922 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:18,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:20,841 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:24,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:24,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:24,297 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:24,302 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:24,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:26,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:10:26,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:10:26,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:10:26,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:26,932 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:29,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:29,210 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:29,210 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:29,215 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:29,934 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:30,936 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:31,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:34,402 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:34,403 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:34,403 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:34,408 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:34,939 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:36,409 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:36,585 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:10:36,942 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:38,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:38,247 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:38,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:38,252 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:38,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:38,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:41,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:10:41,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:10:41,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:10:42,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:44,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:44,265 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:44,265 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:44,270 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:44,948 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:46,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:47,305 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:49,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:49,202 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:49,202 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:49,207 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:49,953 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:50,956 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:53,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:53,074 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:53,074 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:53,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:53,079 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:53,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:54,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:10:56,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:10:56,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:10:56,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:10:58,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:58,142 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:10:58,142 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:58,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:58,147 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:58,966 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:10:58,966 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:01,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:01,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:01,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:01,742 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:01,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:02,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:03,743 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:06,590 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:11:08,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:08,019 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:08,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:08,024 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:08,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:08,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:09,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:11,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:11:11,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:11:11,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:11:11,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:11,563 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:11,563 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:11,568 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:11,977 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:12,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:14,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:15,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:15,539 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:15,539 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:15,544 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:15,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:16,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:19,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:19,354 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:19,354 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:19,359 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:19,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:20,360 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:20,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:25,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:25,129 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:25,130 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:25,135 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:25,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:26,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:11:26,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:11:26,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:11:26,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:26,995 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:31,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:33,862 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:33,862 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:33,863 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:33,867 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:33,999 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:35,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:36,590 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:11:36,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:37,571 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:37,572 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:37,572 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:37,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:38,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:39,005 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:41,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:11:41,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:11:41,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:11:42,203 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:42,203 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:42,203 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:42,204 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:42,208 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:43,010 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:43,010 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:47,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:48,341 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:48,342 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:48,342 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:48,347 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:49,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:51,016 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:52,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:52,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:52,806 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:52,807 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:52,812 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:53,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:55,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:11:56,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:11:56,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:11:56,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:11:56,604 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:56,604 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:11:56,604 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:56,609 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:57,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:11:57,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:59,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:02,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:04,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:04,911 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:04,911 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:04,916 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:05,030 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:06,595 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:12:07,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:08,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:08,251 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:08,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:08,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:08,256 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:09,036 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:09,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:11,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:12:11,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:12:11,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:12:13,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:13,411 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:13,412 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:13,412 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:13,417 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:14,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:15,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:17,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:17,326 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:17,326 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:17,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:18,045 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:18,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:19,047 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:21,691 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:21,692 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:21,692 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:21,697 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:22,049 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:23,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:23,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:25,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:25,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:25,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:25,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:26,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:12:26,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:12:26,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:12:26,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:27,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:29,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:30,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:30,096 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:30,096 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:30,101 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:31,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:31,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:35,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:35,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:35,583 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:35,583 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:35,588 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:36,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:36,594 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:12:37,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:40,596 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:40,926 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:40,927 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:40,927 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:40,932 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:41,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:12:41,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:12:41,032 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:12:41,069 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:43,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:46,196 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:46,196 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:46,197 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:46,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:46,201 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:47,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:47,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:50,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:50,926 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:50,926 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:50,931 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:51,078 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:51,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:53,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:55,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:55,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:12:55,853 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:55,858 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:56,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:12:56,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:12:56,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:12:56,084 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:12:57,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:12:57,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:02,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:02,780 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:02,781 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:02,781 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:02,786 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:03,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:05,093 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:06,596 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:13:06,974 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:06,975 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:06,975 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:06,980 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:07,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:07,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:09,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:11,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:13:11,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:13:11,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:13:11,772 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:11,772 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:11,772 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:11,777 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:12,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:13,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:13,816 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:16,789 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:16,790 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:16,790 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:16,796 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:17,106 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:19,108 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:19,796 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:23,303 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:23,304 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:23,304 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:23,308 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:24,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:25,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:25,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:26,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:13:26,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:13:26,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:13:26,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:26,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:26,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:26,937 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:27,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:29,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:30,632 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:30,633 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:30,633 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:30,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:30,639 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:31,120 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:33,124 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:34,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:34,238 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:34,238 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:34,243 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:35,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:35,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:36,244 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:36,598 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:13:37,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:37,763 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:37,763 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:37,768 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:38,129 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:39,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:41,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:13:41,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:13:41,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:13:41,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:41,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:41,509 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:41,510 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:41,514 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:42,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:43,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:46,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:50,895 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:50,896 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:50,896 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:50,905 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:51,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:51,905 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:53,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:13:56,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:13:56,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:13:56,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:13:56,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:56,968 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:13:56,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:56,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:56,973 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:57,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:13:59,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:00,735 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:00,735 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:00,736 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:00,740 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:01,152 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:02,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:03,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:04,395 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:04,395 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:04,396 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:04,401 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:05,158 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:05,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:06,601 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:14:08,099 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:08,100 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:08,101 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:08,101 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:08,106 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:08,160 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:09,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:11,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:14:11,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:14:11,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:14:11,850 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:11,851 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:11,851 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:11,856 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:12,164 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:13,167 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:13,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:16,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:16,231 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:16,232 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:16,236 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:17,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:17,173 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:19,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:21,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:21,078 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:21,079 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:21,084 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:21,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:23,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:24,928 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:24,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:24,932 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:24,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:24,937 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:25,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:26,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:14:26,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:14:26,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:14:27,182 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:30,087 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:30,088 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:30,088 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:30,088 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:30,093 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:30,185 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:31,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:34,774 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:34,775 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:34,775 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:34,780 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:35,190 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:35,780 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:36,602 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:14:37,193 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:38,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:38,535 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:38,535 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:38,540 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:39,194 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:41,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:14:41,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:14:41,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:14:41,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:41,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:42,542 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:42,542 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:42,542 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:42,547 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:43,198 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:45,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:46,585 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:47,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:47,324 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:47,324 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:47,329 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:48,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:49,205 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:52,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:52,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:52,548 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:52,548 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:52,553 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:53,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:53,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:14:56,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:14:56,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:14:56,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:14:56,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:56,910 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:14:56,910 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:56,915 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:57,213 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:14:57,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:59,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:01,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:01,648 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:01,648 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:01,653 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:02,217 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:03,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:03,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:05,606 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:05,607 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:05,607 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:05,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:06,222 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:06,605 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:15:07,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:09,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:09,368 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:09,368 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:09,368 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:09,373 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:10,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:11,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:15:11,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:15:11,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:15:11,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:13,369 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:13,370 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:13,370 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:13,375 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:14,232 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:14,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:15,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:17,200 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:17,200 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:17,201 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:17,206 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:17,236 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:19,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:20,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:21,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:21,172 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:21,172 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:21,177 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:21,240 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:23,243 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:25,613 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:25,614 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:25,614 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:25,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:25,619 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:26,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:15:26,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:15:26,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:15:26,245 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:27,247 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:30,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:30,581 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:30,581 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:30,586 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:30,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:31,250 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:33,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:35,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:36,610 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:15:36,718 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:36,719 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:36,719 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:36,724 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:37,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:39,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:40,398 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:40,399 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:40,400 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:40,404 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:41,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:15:41,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:15:41,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:15:41,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:41,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:41,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:46,082 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:46,082 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:46,083 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:46,087 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:46,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:47,126 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:47,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:50,101 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:50,102 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:50,102 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:50,107 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:50,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:51,269 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:53,107 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:54,900 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:54,900 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:54,901 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:54,905 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:55,272 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:15:56,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:15:56,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:15:56,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:15:57,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:15:58,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:58,699 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:58,699 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:15:58,700 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:58,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:59,277 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:01,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:02,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:02,670 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:02,670 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:02,675 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:03,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:03,676 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:05,285 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:06,609 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:16:06,892 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:06,893 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:06,893 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:06,898 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:07,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:08,898 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:09,287 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:11,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:16:11,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:16:11,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:16:12,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:12,067 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:12,067 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:12,072 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:12,289 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:13,292 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:14,110 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:16,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:16,054 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:16,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:16,059 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:16,295 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:17,297 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:20,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:20,375 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:20,376 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:20,376 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:20,381 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:21,301 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:21,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:25,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:26,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:16:26,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:16:26,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:16:28,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:28,066 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:28,066 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:28,072 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:28,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:29,309 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:31,110 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:34,662 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:34,662 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:34,663 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:34,667 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:35,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:35,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:36,614 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:16:36,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:38,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:38,895 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:38,895 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:38,900 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:39,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:41,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:16:41,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:16:41,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:16:41,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:42,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:42,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:42,286 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:42,286 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:42,291 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:42,320 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:43,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:47,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:48,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:48,546 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:48,546 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:48,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:49,328 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:49,328 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:52,551 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:55,409 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:55,410 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:16:55,410 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:55,415 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:56,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:16:56,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:16:56,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:16:56,333 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:16:57,335 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:16:58,301 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:00,462 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:00,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:00,463 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:00,468 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:01,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:01,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:03,506 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:05,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:05,569 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:05,569 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:05,574 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:06,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:06,613 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:17:07,345 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:08,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:10,393 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:10,394 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:10,394 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:10,399 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:11,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:17:11,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:17:11,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:17:11,349 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:11,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:13,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:13,673 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:13,673 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:13,674 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:13,678 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:14,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:15,355 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:18,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:18,308 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:18,308 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:18,314 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:18,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:19,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:19,359 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:24,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:24,024 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:24,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:24,029 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:24,363 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:25,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:25,365 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:26,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:17:26,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:17:26,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:17:28,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:28,822 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:28,822 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:28,827 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:29,368 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:30,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:31,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:32,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:32,678 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:32,679 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:32,683 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:33,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:33,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:36,621 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:17:36,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:38,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:38,052 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:38,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:38,057 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:38,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:39,380 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:41,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:17:41,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:17:41,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:17:42,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:46,569 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:46,570 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:46,570 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:46,575 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:47,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:47,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:47,613 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:51,604 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:51,605 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:17:51,605 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:51,610 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:52,390 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:17:53,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:17:53,611 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:56,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:17:56,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:17:56,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:17:59,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:00,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:00,850 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:00,850 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:00,855 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:01,398 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:03,401 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:04,603 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:04,604 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:04,604 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:04,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:04,610 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:05,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:05,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:06,617 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:18:09,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:10,163 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:10,164 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:10,164 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:10,169 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:10,407 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:11,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:18:11,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:18:11,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:18:11,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:14,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:14,542 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:14,542 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:14,547 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:15,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:15,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:15,585 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:18,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:18,937 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:18,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:18,943 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:19,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:20,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:21,420 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:23,452 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:23,453 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:23,453 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:23,459 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:24,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:25,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:26,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:18:26,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:18:26,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:18:26,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:26,895 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:26,896 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:26,896 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:26,901 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:27,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:29,429 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:31,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:32,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:32,590 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:32,590 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:32,595 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:33,433 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:33,433 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:36,619 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:18:37,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:38,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:38,934 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:38,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:38,939 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:39,437 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:41,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:18:41,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:18:41,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:18:41,440 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:42,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:42,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:42,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:42,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:42,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:43,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:43,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:47,781 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:51,249 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:51,250 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:51,250 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:51,255 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:51,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:53,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:53,450 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:56,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:56,028 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:56,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:56,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:18:56,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:18:56,034 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:56,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:18:56,453 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:18:57,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:18:58,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:59,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:59,615 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:18:59,615 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:59,620 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:00,458 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:01,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:03,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:05,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:05,016 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:05,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:05,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:05,463 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:06,622 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:19:07,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:08,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:08,965 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:08,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:08,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:08,970 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:09,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:11,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:19:11,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:19:11,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:19:11,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:14,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:17,536 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:17,537 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:17,537 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:17,542 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:18,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:19,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:19,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:23,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:23,819 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:23,820 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:23,825 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:24,481 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:24,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:25,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:26,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:19:26,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:19:26,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:19:28,085 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:28,086 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:28,086 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:28,091 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:28,486 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:29,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:30,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:33,778 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:33,779 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:33,779 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:33,784 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:34,492 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:35,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:35,785 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:36,624 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:19:38,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:38,310 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:38,310 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:38,315 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:38,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:39,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:41,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:19:41,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:19:41,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:19:41,340 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:42,583 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:42,584 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:42,584 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:42,589 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:43,503 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:43,503 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:46,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:51,125 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:51,126 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:51,126 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:51,132 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:51,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:52,132 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:53,511 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:55,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:55,518 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:55,518 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:55,523 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:56,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:19:56,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:19:56,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:19:56,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:19:57,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:57,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:19:59,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:59,249 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:19:59,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:59,254 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:59,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:01,520 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:02,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:03,551 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:03,552 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:03,552 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:03,557 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:04,522 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:05,524 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:06,626 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:20:07,135 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:07,136 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:07,136 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:07,141 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:07,525 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:08,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:09,529 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:11,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:20:11,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:20:11,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:20:11,161 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:11,270 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:11,270 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:11,275 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:11,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:13,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:13,533 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:14,912 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:14,913 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:14,913 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:14,921 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:15,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:15,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:18,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:19,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:19,058 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:19,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:19,063 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:19,540 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:21,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:23,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:23,297 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:23,297 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:23,302 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:23,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:24,303 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:25,548 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:26,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:20:26,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:20:26,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:20:27,643 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:27,644 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:27,644 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:27,649 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:28,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:29,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:29,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:31,440 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:31,440 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:31,441 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:31,445 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:31,554 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:33,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:35,446 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:36,628 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:20:37,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:37,702 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:37,702 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:37,707 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:38,560 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:39,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:40,708 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:41,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:20:41,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:20:41,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:20:42,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:42,724 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:42,724 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:42,729 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:43,566 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:43,567 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:45,765 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:47,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:47,648 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:47,648 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:47,653 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:48,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:49,571 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:51,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:51,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:51,754 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:51,754 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:51,759 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:52,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:53,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:20:56,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:20:56,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:20:56,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:20:56,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:56,307 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:20:56,307 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:56,312 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:56,578 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:20:57,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:57,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:00,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:00,052 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:00,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:00,057 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:00,583 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:01,585 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:03,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:03,732 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:03,733 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:03,733 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:03,738 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:04,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:05,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:06,633 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:21:08,391 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:08,391 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:08,392 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:08,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:08,396 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:08,591 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:09,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:11,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:21:11,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:21:11,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:21:12,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:12,498 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:12,498 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:12,502 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:12,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:13,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:13,598 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:16,548 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:16,548 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:16,549 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:16,553 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:16,601 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:17,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:18,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:22,443 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:22,444 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:22,444 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:22,449 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:22,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:23,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:24,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:26,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:21:26,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:21:26,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:21:26,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:26,734 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:26,734 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:26,739 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:27,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:27,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:29,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:30,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:30,368 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:30,369 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:30,374 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:30,615 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:31,616 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:35,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:35,336 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:35,336 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:35,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:35,342 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:35,618 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:36,632 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:21:37,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:40,487 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:40,488 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:40,488 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:40,488 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:40,493 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:40,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:41,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:21:41,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:21:41,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:21:41,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:46,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:46,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:46,471 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:46,471 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:46,476 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:46,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:47,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:50,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:50,541 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:50,542 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:50,546 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:50,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:51,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:51,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:54,628 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:54,629 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:54,629 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:54,634 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:54,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:55,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:21:56,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:21:56,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:21:56,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:21:57,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:58,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:58,561 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:21:58,561 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:58,566 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:58,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:21:59,645 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:02,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:04,390 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:04,391 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:04,392 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:04,396 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:04,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:05,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:06,637 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:22:07,638 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:08,068 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:08,069 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:08,070 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:08,074 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:08,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:09,655 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:11,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:22:11,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:22:11,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:22:12,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:12,055 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:12,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:12,060 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:12,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:13,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:13,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:16,891 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:16,892 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:16,893 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:16,897 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:17,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:17,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:18,898 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:21,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:21,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:21,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:21,516 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:21,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:23,670 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:24,517 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:26,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:22:26,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:22:26,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:22:26,388 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:26,389 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:26,389 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:26,394 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:26,672 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:27,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:30,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:31,135 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:31,136 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:31,136 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:31,141 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:31,678 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:33,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:34,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:34,960 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:34,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:34,965 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:35,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:35,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:35,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:36,636 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:22:39,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:39,915 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:39,915 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:39,920 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:40,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:41,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:22:41,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:22:41,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:22:41,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:41,689 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:46,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:46,416 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:46,416 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:46,416 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:46,421 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:46,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:47,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:50,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:50,052 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:50,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:50,057 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:50,696 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:22:51,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:22:52,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:56,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:22:56,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:22:56,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:22:57,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:59,199 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:59,200 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:22:59,200 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:59,205 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:59,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:01,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:03,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:05,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:05,439 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:05,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:05,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:05,710 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:06,641 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:23:07,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:08,643 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:09,811 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:09,811 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:09,812 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:09,816 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:10,714 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:11,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:23:11,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:23:11,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:23:11,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:14,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:18,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:18,252 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:18,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:18,257 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:18,722 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:19,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:19,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:22,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:22,323 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:22,323 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:22,328 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:22,727 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:23,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:24,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:25,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:25,894 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:25,894 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:25,898 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:26,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:23:26,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:23:26,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:23:26,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:27,734 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:30,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:30,445 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:30,446 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:30,446 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:30,451 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:30,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:31,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:35,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:35,029 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:35,030 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:35,034 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:35,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:35,743 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:36,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:36,640 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:23:39,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:39,836 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:39,836 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:39,841 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:40,745 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:41,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:23:41,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:23:41,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:23:41,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:41,748 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:44,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:44,938 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:44,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:44,943 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:45,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:45,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:46,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:49,087 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:49,088 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:49,088 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:49,093 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:49,754 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:49,754 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:52,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:56,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:23:56,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:23:56,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:23:56,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:56,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:56,325 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:56,330 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:56,759 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:23:57,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:57,762 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:23:59,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:59,854 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:23:59,854 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:59,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:00,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:01,766 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:02,860 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:03,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:03,491 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:03,491 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:03,496 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:03,767 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:05,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:06,642 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:24:08,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:11,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:24:11,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:24:11,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:24:11,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:11,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:11,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:11,939 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:12,774 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:13,777 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:13,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:17,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:17,648 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:17,648 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:17,653 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:17,780 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:19,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:19,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:21,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:21,475 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:21,475 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:21,480 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:21,785 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:23,789 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:25,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:25,637 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:25,638 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:25,638 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:25,644 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:25,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:26,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:24:26,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:24:26,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:24:27,793 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:30,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:30,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:30,853 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:30,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:30,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:31,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:31,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:35,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:35,035 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:35,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:35,040 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:35,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:35,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:36,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:36,644 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:24:40,386 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:40,387 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:40,387 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:40,392 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:40,804 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:41,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:24:41,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:24:41,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:24:41,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:41,807 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:45,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:45,464 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:45,464 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:45,469 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:45,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:46,505 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:47,812 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:49,085 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:49,086 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:49,086 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:49,091 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:49,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:51,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:52,091 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:52,912 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:52,913 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:52,913 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:52,918 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:53,818 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:55,821 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:24:56,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:24:56,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:24:56,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:24:56,863 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:56,863 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:24:56,864 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:56,868 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:57,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:24:57,905 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:59,826 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:00,397 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:00,397 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:00,398 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:00,402 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:00,827 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:01,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:03,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:04,639 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:04,640 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:04,640 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:04,645 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:04,829 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:05,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:06,647 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:25:08,572 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:08,573 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:08,573 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:08,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:08,578 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:08,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:09,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:11,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:25:11,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:25:11,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:25:12,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:12,649 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:12,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:12,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:12,838 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:13,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:13,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:16,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:16,478 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:16,478 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:16,483 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:16,843 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:17,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:19,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:23,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:23,065 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:23,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:23,070 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:23,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:25,071 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:25,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:26,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:25:26,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:25:26,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:25:27,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:27,833 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:27,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:27,838 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:27,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:29,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:30,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:32,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:32,440 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:32,440 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:32,445 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:32,859 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:33,862 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:36,446 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:36,460 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:36,461 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:36,461 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:36,466 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:36,649 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:25:36,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:37,866 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:41,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:25:41,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:25:41,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:25:42,120 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:42,121 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:42,121 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:42,121 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:42,126 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:42,869 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:43,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:46,529 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:46,530 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:46,530 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:46,535 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:46,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:47,536 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:47,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:50,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:50,248 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:50,248 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:50,253 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:50,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:51,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:53,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:54,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:54,259 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:54,259 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:54,264 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:54,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:55,885 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:25:56,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:25:56,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:25:56,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:25:58,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:58,382 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:25:58,382 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:58,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:58,387 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:58,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:25:59,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:01,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:01,940 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:01,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:01,945 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:02,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:03,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:03,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:05,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:05,866 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:05,866 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:05,871 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:05,895 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:06,651 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:26:07,898 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:09,586 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:09,587 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:09,587 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:09,587 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:09,592 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:09,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:11,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:26:11,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:26:11,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:26:11,902 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:12,976 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:12,977 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:12,977 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:12,982 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:13,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:15,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:15,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:20,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:21,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:21,428 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:21,428 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:21,433 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:21,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:23,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:25,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:25,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:25,727 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:25,727 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:25,732 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:25,916 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:26,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:26:26,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:26:26,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:26:27,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:30,341 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:30,341 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:30,342 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:30,346 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:30,921 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:31,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:31,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:33,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:33,972 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:33,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:33,977 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:34,926 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:35,928 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:36,653 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:26:36,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:37,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:37,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:37,695 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:37,700 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:37,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:39,932 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:41,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:26:41,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:26:41,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:26:41,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:41,874 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:41,874 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:41,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:41,879 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:41,934 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:43,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:46,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:47,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:47,562 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:47,562 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:47,567 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:47,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:49,940 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:51,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:51,364 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:51,364 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:51,369 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:51,941 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:52,369 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:53,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:55,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:55,824 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:26:55,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:55,829 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:55,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:26:56,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:26:56,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:26:56,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:26:57,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:26:58,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:00,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:00,555 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:00,555 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:00,560 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:00,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:01,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:03,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:05,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:05,037 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:05,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:05,042 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:05,956 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:06,655 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:27:07,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:08,656 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:10,544 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:10,545 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:10,545 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:10,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:10,961 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:11,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:27:11,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:27:11,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:27:11,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:14,301 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:14,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:14,726 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:14,726 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:14,731 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:14,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:15,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:18,724 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:18,726 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:18,726 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:18,731 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:18,971 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:19,731 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:19,972 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:24,732 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:24,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:24,767 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:24,768 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:24,774 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:24,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:25,977 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:26,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:27:26,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:27:26,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:27:29,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:29,320 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:29,320 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:29,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:29,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:30,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:31,983 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:33,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:33,777 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:33,778 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:33,782 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:33,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:35,783 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:35,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:36,660 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:27:38,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:38,018 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:38,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:38,023 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:38,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:39,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:41,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:41,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:27:41,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:27:41,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:27:43,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:43,988 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:43,988 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:43,993 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:43,994 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:45,997 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:46,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:48,347 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:48,348 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:48,348 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:48,353 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:48,999 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:50,002 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:51,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:52,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:52,053 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:52,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:52,058 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:53,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:54,006 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:55,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:55,946 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:55,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:55,951 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:56,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:27:56,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:27:56,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:27:56,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:27:57,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:58,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:27:59,716 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:59,717 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:27:59,717 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:59,722 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:00,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:02,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:02,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:03,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:03,439 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:03,440 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:03,445 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:04,016 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:06,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:06,659 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:28:08,660 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:10,132 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:10,133 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:10,133 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:10,138 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:11,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:11,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:28:11,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:28:11,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:28:12,025 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:14,311 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:14,593 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:14,593 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:14,593 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:14,598 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:15,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:16,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:18,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:18,318 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:18,318 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:18,323 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:19,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:19,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:20,036 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:24,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:26,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:28:26,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:28:26,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:28:27,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:27,539 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:27,539 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:27,543 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:28,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:29,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:30,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:30,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:30,838 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:30,839 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:30,844 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:31,045 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:32,048 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:34,844 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:34,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:34,894 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:34,894 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:34,899 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:35,049 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:36,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:36,664 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:28:38,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:38,642 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:38,642 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:38,647 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:39,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:40,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:40,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:41,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:28:41,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:28:41,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:28:42,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:42,645 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:42,645 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:42,650 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:43,058 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:44,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:45,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:47,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:47,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:47,462 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:47,467 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:48,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:50,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:51,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:51,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:51,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:51,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:51,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:52,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:54,071 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:28:55,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:55,557 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:28:55,557 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:55,562 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:56,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:28:56,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:28:56,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:28:56,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:28:57,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:58,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:02,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:04,215 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:04,215 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:04,216 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:04,220 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:05,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:06,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:06,663 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:29:07,664 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:10,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:10,010 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:10,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:10,015 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:10,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:11,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:29:11,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:29:11,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:29:12,088 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:13,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:13,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:13,981 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:13,981 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:13,986 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:14,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:16,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:18,145 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:18,145 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:18,146 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:18,151 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:19,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:19,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:20,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:22,794 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:22,795 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:22,795 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:22,800 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:23,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:24,103 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:24,801 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:26,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:29:26,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:29:26,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:29:26,982 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:26,983 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:26,983 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:26,988 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:27,105 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:28,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:30,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:30,240 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:30,241 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:30,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:30,246 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:31,110 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:32,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:34,400 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:34,401 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:34,401 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:34,406 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:35,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:35,407 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:36,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:36,668 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:29:38,606 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:38,607 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:38,607 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:38,613 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:39,118 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:40,120 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:40,613 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:41,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:29:41,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:29:41,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:29:45,077 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:45,078 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:45,078 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:45,083 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:45,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:46,121 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:48,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:48,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:48,954 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:48,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:48,960 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:49,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:50,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:51,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:52,671 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:52,672 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:52,672 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:52,677 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:53,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:29:54,134 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:29:56,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:29:56,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:29:56,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:29:57,494 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:57,495 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:29:57,495 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:57,495 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:57,500 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:58,138 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:00,141 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:02,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:02,236 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:02,236 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:02,241 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:03,143 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:03,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:04,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:06,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:06,253 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:06,253 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:06,258 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:06,667 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:30:07,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:08,149 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:08,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:10,409 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:10,410 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:10,410 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:10,415 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:11,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:30:11,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:30:11,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:30:11,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:12,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:14,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:18,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:18,950 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:18,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:18,955 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:19,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:19,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:20,162 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:22,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:22,362 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:22,362 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:22,367 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:23,164 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:24,167 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:25,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:26,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:30:26,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:30:26,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:30:26,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:26,585 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:26,585 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:26,590 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:27,169 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:28,171 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:30,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:33,436 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:33,436 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:33,437 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:33,442 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:34,175 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:36,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:36,442 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:36,672 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:30:37,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:37,003 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:37,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:37,008 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:37,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:38,181 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:41,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:30:41,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:30:41,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:30:41,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:41,294 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:41,295 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:41,299 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:42,183 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:42,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:44,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:47,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:48,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:48,058 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:48,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:48,063 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:48,188 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:50,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:53,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:53,006 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:53,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:53,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:53,011 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:53,193 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:54,196 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:30:56,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:30:56,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:30:56,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:30:57,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:57,011 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:30:57,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:57,016 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:57,199 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:30:58,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:58,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:01,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:01,063 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:01,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:01,069 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:01,203 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:02,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:03,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:05,732 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:05,732 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:05,733 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:05,737 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:06,208 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:06,672 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:31:08,211 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:08,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:09,070 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:09,071 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:09,071 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:09,076 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:09,212 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:10,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:11,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:31:11,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:31:11,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:31:12,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:12,803 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:12,803 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:12,808 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:13,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:13,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:14,219 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:17,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:17,569 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:17,569 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:17,575 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:18,223 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:19,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:20,226 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:21,187 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:21,188 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:21,188 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:21,193 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:21,226 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:22,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:25,193 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:25,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:25,830 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:25,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:25,835 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:26,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:31:26,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:31:26,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:31:26,232 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:28,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:30,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:32,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:32,725 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:32,726 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:32,730 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:33,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:34,241 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:35,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:36,674 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:31:39,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:39,115 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:39,115 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:39,121 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:39,243 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:40,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:41,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:31:41,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:31:41,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:31:41,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:43,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:43,012 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:43,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:43,018 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:43,248 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:44,250 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:47,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:47,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:47,422 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:47,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:47,427 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:48,252 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:50,255 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:52,428 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:52,986 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:52,987 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:52,987 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:52,992 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:53,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:54,259 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:31:56,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:31:56,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:31:56,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:31:56,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:56,918 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:31:56,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:56,923 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:57,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:31:57,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:58,265 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:02,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:05,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:05,261 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:05,262 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:05,266 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:05,269 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:06,676 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:32:08,273 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:08,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:10,354 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:10,354 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:10,355 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:10,359 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:11,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:32:11,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:32:11,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:32:11,274 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:12,277 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:14,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:14,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:14,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:14,523 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:14,527 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:15,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:16,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:19,565 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:21,384 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:21,385 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:21,385 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:21,390 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:22,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:24,289 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:25,391 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:26,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:32:26,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:32:26,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:32:28,188 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:28,188 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:28,189 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:28,193 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:28,291 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:30,295 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:31,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:32,564 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:32,565 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:32,565 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:32,570 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:33,297 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:34,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:36,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:36,678 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:32:38,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:38,098 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:38,099 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:38,104 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:38,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:40,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:41,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:32:41,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:32:41,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:32:42,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:43,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:43,949 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:43,949 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:43,954 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:44,308 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:46,311 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:47,873 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:47,874 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:47,874 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:47,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:47,879 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:48,312 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:50,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:52,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:52,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:52,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:52,473 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:53,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:53,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:54,320 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:56,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:32:56,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:32:56,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:32:57,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:57,250 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:32:57,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:57,255 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:57,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:32:58,325 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:32:59,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:01,001 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:01,005 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:01,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:01,010 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:01,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:02,330 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:05,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:05,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:05,422 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:05,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:05,427 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:06,332 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:06,680 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:33:08,335 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:10,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:11,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:33:11,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:33:11,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:33:13,803 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:13,804 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:13,804 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:13,809 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:14,339 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:15,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:16,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:18,687 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:18,688 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:18,688 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:18,694 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:19,345 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:20,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:21,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:24,345 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:24,346 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:24,346 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:24,351 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:24,352 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:26,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:33:26,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:33:26,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:33:26,355 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:27,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:29,557 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:29,557 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:29,558 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:29,562 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:30,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:32,361 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:32,601 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:36,682 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:33:36,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:36,823 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:36,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:36,829 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:37,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:37,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:38,366 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:41,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:33:41,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:33:41,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:33:41,368 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:41,369 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:41,369 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:41,374 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:42,369 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:43,413 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:44,372 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:46,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:46,434 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:46,434 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:46,439 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:47,374 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:48,376 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:48,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:52,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:52,131 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:52,131 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:52,136 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:52,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:54,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:54,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:56,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:33:56,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:33:56,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:33:56,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:56,349 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:33:56,349 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:56,354 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:56,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:33:58,385 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:33:59,393 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:01,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:01,303 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:01,303 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:01,308 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:01,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:02,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:04,649 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:04,650 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:04,650 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:04,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:04,655 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:05,395 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:06,397 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:06,688 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:34:09,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:10,640 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:10,641 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:10,641 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:10,646 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:11,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:34:11,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:34:11,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:34:11,403 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:12,406 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:15,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:15,492 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:15,493 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:15,493 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:15,498 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:16,410 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:18,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:19,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:19,407 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:19,407 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:19,412 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:19,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:20,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:22,420 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:22,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:22,993 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:22,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:22,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:23,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:24,423 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:25,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:26,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:34:26,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:34:26,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:34:26,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:26,790 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:26,791 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:26,795 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:27,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:28,429 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:30,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:30,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:30,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:30,443 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:31,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:31,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:32,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:36,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:36,686 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:34:38,979 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:38,979 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:38,980 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:38,985 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:39,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:40,445 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:41,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:34:41,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:34:41,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:34:42,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:47,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:47,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:47,339 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:47,339 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:47,344 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:47,454 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:48,457 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:52,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:55,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:55,654 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:55,654 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:55,659 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:56,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:34:56,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:34:56,040 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:34:56,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:34:58,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:58,471 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:34:59,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:59,590 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:34:59,591 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:59,595 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:00,472 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:02,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:03,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:05,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:05,826 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:05,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:05,831 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:06,481 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:06,691 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:35:08,486 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:08,693 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:11,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:11,037 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:11,037 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:11,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:35:11,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:35:11,042 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:11,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:35:11,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:12,491 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:14,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:15,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:15,859 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:15,860 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:15,865 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:16,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:18,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:19,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:20,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:20,045 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:20,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:20,050 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:20,500 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:22,505 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:25,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:25,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:25,078 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:25,079 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:25,083 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:25,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:26,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:35:26,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:35:26,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:35:26,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:29,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:29,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:29,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:29,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:29,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:30,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:30,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:35,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:35,322 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:35,322 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:35,327 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:35,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:36,327 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:36,525 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:36,691 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:35:40,365 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:40,365 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:40,366 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:40,370 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:40,529 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:41,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:35:41,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:35:41,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:35:42,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:42,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:45,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:45,331 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:45,331 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:45,335 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:45,537 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:46,540 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:47,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:50,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:50,287 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:50,287 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:50,292 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:50,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:52,548 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:53,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:55,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:55,305 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:35:55,305 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:55,310 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:55,551 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:35:56,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:35:56,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:35:56,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:35:56,554 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:35:58,347 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:01,099 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:01,099 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:01,099 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:01,104 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:01,558 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:02,560 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:04,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:04,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:04,834 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:04,834 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:04,839 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:05,566 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:06,568 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:06,693 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:36:09,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:09,888 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:09,889 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:09,889 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:09,894 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:10,572 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:11,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:36:11,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:36:11,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:36:12,575 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:15,081 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:15,085 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:15,085 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:15,085 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:15,090 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:15,582 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:16,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:19,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:19,935 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:19,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:19,940 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:20,586 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:20,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:22,591 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:24,110 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:24,110 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:24,111 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:24,116 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:24,593 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:26,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:36:26,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:36:26,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:36:26,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:26,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:27,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:27,734 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:27,734 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:27,739 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:28,598 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:30,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:31,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:32,764 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:32,765 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:32,765 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:32,770 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:33,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:34,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:36,695 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:36:36,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:36,965 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:36,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:36,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:36,969 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:37,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:38,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:40,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:40,758 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:40,758 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:40,763 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:41,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:36:41,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:36:41,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:36:41,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:42,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:42,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:45,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:45,791 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:45,791 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:45,795 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:46,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:47,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:48,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:52,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:54,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:54,330 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:54,330 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:54,335 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:54,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:36:56,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:36:56,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:36:56,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:36:56,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:36:57,843 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:57,843 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:36:57,844 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:57,844 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:57,849 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:58,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:00,645 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:01,720 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:01,721 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:01,721 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:01,726 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:02,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:03,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:04,651 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:06,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:06,166 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:06,167 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:06,171 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:06,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:06,697 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:37:08,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:09,699 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:11,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:37:11,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:37:11,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:37:14,656 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:14,657 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:14,657 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:14,662 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:14,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:15,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:16,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:20,631 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:20,632 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:20,632 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:20,637 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:20,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:21,637 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:22,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:25,787 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:25,788 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:25,788 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:25,793 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:26,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:37:26,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:37:26,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:37:26,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:27,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:28,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:30,610 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:30,614 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:30,614 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:30,618 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:30,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:32,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:32,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:36,699 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:37:37,700 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:39,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:39,237 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:39,238 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:39,242 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:39,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:40,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:41,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:37:41,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:37:41,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:37:43,343 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:43,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:43,947 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:43,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:43,953 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:44,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:46,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:48,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:49,327 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:49,328 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:49,328 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:49,333 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:49,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:50,715 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:53,586 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:53,587 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:53,587 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:53,592 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:53,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:54,593 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:54,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:37:56,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:37:56,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:37:56,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:37:57,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:57,157 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:37:57,157 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:57,162 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:57,724 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:37:58,726 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:00,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:01,035 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:01,036 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:01,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:01,041 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:01,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:02,732 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:05,184 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:05,184 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:05,185 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:05,190 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:05,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:06,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:06,701 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:38:06,738 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:10,416 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:10,417 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:10,417 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:10,422 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:10,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:11,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:38:11,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:38:11,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:38:11,345 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:12,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:14,454 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:14,455 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:14,455 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:14,460 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:14,748 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:16,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:16,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:19,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:19,822 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:19,823 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:19,827 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:20,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:21,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:22,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:23,840 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:23,841 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:23,841 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:23,846 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:24,762 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:26,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:38:26,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:38:26,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:38:26,766 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:27,336 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:28,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:28,238 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:28,238 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:28,243 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:28,767 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:30,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:33,282 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:34,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:34,500 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:34,500 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:34,505 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:34,776 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:36,707 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:38:36,780 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:38,511 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:38,512 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:38,512 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:38,512 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:38,517 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:38,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:40,786 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:41,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:38:41,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:38:41,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:38:42,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:42,142 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:42,142 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:42,147 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:42,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:44,185 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:44,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:49,185 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:50,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:50,553 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:50,554 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:50,559 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:50,798 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:52,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:54,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:56,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:38:56,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:38:56,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:38:56,671 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:56,672 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:38:56,673 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:56,677 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:56,806 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:38:58,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:38:59,713 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:01,870 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:01,870 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:01,871 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:01,876 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:02,815 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:04,820 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:04,876 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:06,705 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:39:08,215 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:08,216 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:08,216 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:08,221 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:08,824 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:10,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:10,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:11,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:39:11,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:39:11,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:39:12,184 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:12,184 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:12,185 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:12,189 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:12,830 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:14,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:15,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:16,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:16,750 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:16,751 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:16,756 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:16,837 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:18,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:20,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:20,137 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:20,137 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:20,142 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:20,841 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:21,143 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:22,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:25,397 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:25,398 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:25,398 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:25,404 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:25,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:26,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:39:26,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:39:26,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:39:26,268 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:26,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:30,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:30,464 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:30,464 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:30,469 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:30,855 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:31,505 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:32,860 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:34,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:34,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:34,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:34,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:35,863 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:36,710 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:39:36,711 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:36,866 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:38,718 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:38,719 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:38,719 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:38,724 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:38,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:40,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:41,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:39:41,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:39:41,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:39:42,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:42,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:42,339 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:42,339 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:42,344 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:42,872 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:44,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:46,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:46,095 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:46,095 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:46,100 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:46,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:48,101 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:48,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:52,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:52,155 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:52,155 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:52,160 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:52,886 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:53,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:54,891 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:39:56,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:39:56,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:39:56,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:39:57,855 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:57,856 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:39:57,856 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:57,862 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:57,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:39:58,897 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:00,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:02,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:02,881 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:02,881 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:02,886 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:02,900 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:04,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:04,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:06,709 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:40:07,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:07,940 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:07,941 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:07,946 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:08,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:09,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:10,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:11,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:40:11,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:40:11,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:40:13,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:13,046 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:13,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:13,051 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:13,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:14,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:15,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:16,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:16,829 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:16,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:16,834 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:16,922 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:18,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:20,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:21,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:21,834 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:21,834 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:21,839 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:21,928 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:24,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:25,840 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:26,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:40:26,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:40:26,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:40:27,539 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:27,539 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:27,539 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:27,544 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:27,936 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:28,939 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:31,126 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:31,127 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:31,127 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:31,128 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:31,133 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:31,943 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:32,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:34,572 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:34,573 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:34,573 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:34,579 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:34,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:36,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:36,715 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:40:36,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:40,913 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:40,914 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:40,914 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:40,919 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:40,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:41,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:40:41,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:40:41,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:40:42,336 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:42,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:45,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:45,094 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:45,094 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:45,099 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:45,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:46,965 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:48,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:48,688 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:48,689 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:48,689 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:48,694 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:48,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:50,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:51,997 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:51,998 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:51,998 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:52,003 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:52,971 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:54,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:54,976 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:56,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:40:56,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:40:56,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:40:57,088 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:57,089 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:40:57,089 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:57,094 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:57,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:40:58,981 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:40:59,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:00,671 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:00,672 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:00,672 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:00,677 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:00,983 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:02,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:04,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:05,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:05,564 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:05,564 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:05,570 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:05,991 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:06,713 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:41:06,994 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:09,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:09,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:09,745 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:09,745 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:09,750 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:09,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:10,999 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:11,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:41:11,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:41:11,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:41:13,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:13,615 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:13,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:13,621 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:14,002 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:15,005 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:15,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:17,595 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:17,596 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:17,596 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:17,601 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:18,008 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:19,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:21,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:21,764 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:21,765 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:21,765 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:21,770 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:22,014 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:23,016 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:26,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:41:26,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:41:26,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:41:26,204 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:26,317 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:26,317 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:26,322 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:27,020 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:27,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:29,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:32,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:33,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:33,704 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:33,704 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:33,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:34,030 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:35,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:36,719 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:41:37,721 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:38,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:38,307 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:38,307 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:38,312 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:39,036 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:41,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:41,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:41:41,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:41:41,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:41:43,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:46,792 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:46,793 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:46,793 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:46,798 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:47,047 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:48,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:49,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:51,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:51,265 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:51,265 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:51,270 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:52,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:53,058 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:41:54,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:55,899 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:55,900 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:41:55,900 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:55,905 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:56,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:41:56,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:41:56,044 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:41:56,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:41:57,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:00,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:00,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:00,305 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:00,305 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:00,310 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:01,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:03,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:05,345 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:06,718 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:42:06,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:06,809 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:06,810 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:06,816 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:07,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:09,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:10,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:10,618 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:10,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:10,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:10,624 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:11,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:42:11,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:42:11,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:42:11,082 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:13,087 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:15,131 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:15,132 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:15,132 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:15,137 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:16,091 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:16,173 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:17,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:18,706 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:18,707 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:18,708 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:18,712 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:19,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:21,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:21,713 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:23,282 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:23,283 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:23,284 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:23,288 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:24,101 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:25,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:26,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:42:26,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:42:26,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:42:26,710 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:26,711 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:26,711 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:26,716 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:26,716 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:27,105 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:29,109 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:31,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:34,203 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:34,204 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:34,204 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:34,209 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:35,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:36,720 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:42:37,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:37,721 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:38,200 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:38,201 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:38,201 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:38,207 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:39,117 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:41,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:42:41,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:42:41,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:42:41,120 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:41,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:41,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:41,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:41,997 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:42,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:43,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:45,124 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:45,928 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:45,929 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:45,929 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:45,935 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:46,125 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:47,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:48,935 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:51,754 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:51,758 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:51,759 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:51,764 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:52,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:53,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:42:54,765 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:56,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:42:56,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:42:56,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:42:56,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:56,328 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:42:56,328 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:56,333 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:57,136 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:42:59,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:00,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:00,152 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:00,152 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:00,152 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:00,157 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:01,141 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:03,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:03,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:03,935 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:03,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:03,940 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:04,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:05,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:05,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:06,722 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:43:07,927 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:07,927 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:07,928 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:07,932 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:08,149 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:09,152 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:11,042 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:43:11,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:43:11,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:43:11,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:16,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:16,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:16,347 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:16,347 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:16,352 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:17,158 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:19,161 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:21,390 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:22,165 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:22,166 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:22,166 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:22,171 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:23,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:25,167 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:26,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:43:26,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:43:26,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:43:27,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:27,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:27,681 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:27,681 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:27,686 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:28,169 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:29,171 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:31,476 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:31,477 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:31,477 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:31,482 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:32,173 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:32,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:33,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:35,978 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:35,979 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:35,980 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:35,984 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:36,177 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:36,724 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:43:37,180 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:37,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:41,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:43:41,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:43:41,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:43:43,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:44,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:44,190 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:44,190 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:44,195 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:45,184 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:47,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:49,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:49,491 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:49,492 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:49,492 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:49,497 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:50,189 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:51,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:54,498 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:55,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:55,015 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:55,015 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:55,020 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:55,194 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:43:56,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:43:56,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:43:56,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:43:57,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:43:59,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:59,682 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:43:59,683 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:59,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:59,687 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:00,200 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:01,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:04,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:06,729 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:44:07,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:07,994 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:07,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:07,999 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:08,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:09,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:09,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:11,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:44:11,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:44:11,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:44:15,340 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:16,457 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:16,457 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:16,458 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:16,463 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:17,215 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:19,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:19,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:19,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:19,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:19,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:20,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:20,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:21,219 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:24,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:24,275 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:24,275 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:24,280 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:25,222 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:26,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:44:26,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:44:26,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:44:26,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:27,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:28,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:28,090 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:28,090 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:28,095 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:28,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:31,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:32,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:32,977 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:32,978 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:32,978 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:32,983 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:33,230 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:35,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:36,728 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:44:37,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:38,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:38,440 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:38,440 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:38,445 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:39,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:41,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:44:41,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:44:41,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:44:41,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:42,918 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:42,919 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:42,919 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:42,919 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:42,924 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:43,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:45,243 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:47,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:51,139 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:51,140 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:51,141 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:51,145 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:51,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:53,146 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:53,249 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:54,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:54,882 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:54,882 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:54,887 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:55,250 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:44:56,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:44:56,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:44:56,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:44:57,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:44:58,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:58,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:58,826 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:44:58,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:58,831 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:59,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:01,258 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:02,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:02,846 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:02,846 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:02,851 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:03,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:03,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:05,263 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:06,733 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:45:06,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:06,854 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:06,854 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:06,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:07,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:08,860 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:09,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:11,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:45:11,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:45:11,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:45:13,585 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:13,586 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:13,586 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:13,591 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:14,271 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:14,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:15,273 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:17,435 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:17,436 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:17,436 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:17,441 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:18,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:19,279 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:20,442 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:25,442 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:25,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:25,646 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:25,647 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:25,651 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:26,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:45:26,044 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:45:26,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:45:26,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:27,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:30,640 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:30,641 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:30,641 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:30,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:30,647 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:31,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:33,291 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:34,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:34,563 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:34,564 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:34,568 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:35,292 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:36,569 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:36,732 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:45:37,295 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:38,511 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:38,512 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:38,512 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:38,517 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:39,297 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:41,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:45:41,044 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:45:41,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:45:41,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:42,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:42,050 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:42,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:42,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:42,054 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:42,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:43,303 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:47,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:47,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:47,825 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:47,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:47,830 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:48,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:49,308 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:45:52,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:56,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:45:56,044 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:45:56,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:45:56,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:56,322 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:45:56,323 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:56,328 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:57,313 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:45:58,365 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:59,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:03,365 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:04,649 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:04,650 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:04,650 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:04,655 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:05,320 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:06,737 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:46:07,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:08,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:08,617 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:08,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:08,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:08,623 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:09,325 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:11,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:46:11,044 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:46:11,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:46:11,328 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:13,534 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:13,535 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:13,535 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:13,540 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:14,330 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:14,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:15,333 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:19,134 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:19,134 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:19,135 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:19,139 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:19,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:20,140 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:21,339 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:23,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:23,252 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:23,252 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:23,257 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:23,340 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:25,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:25,344 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:26,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:46:26,044 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:46:26,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:46:27,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:27,356 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:27,356 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:27,361 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:28,346 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:29,349 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:30,397 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:33,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:33,060 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:33,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:33,066 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:33,351 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:35,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:36,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:36,736 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:46:37,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:37,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:37,259 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:37,264 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:37,355 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:39,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:41,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:46:41,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:46:41,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:46:41,092 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:41,271 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:41,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:41,272 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:41,277 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:41,359 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:43,362 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:45,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:45,161 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:45,162 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:45,166 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:45,363 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:47,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:47,366 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:49,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:49,237 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:49,238 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:49,243 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:49,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:51,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:52,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:53,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:53,049 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:53,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:53,054 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:53,372 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:46:55,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:46:56,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:46:56,044 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:46:56,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:46:57,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:59,187 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:59,188 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:46:59,188 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:59,193 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:59,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:01,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:03,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:03,212 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:03,213 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:03,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:03,218 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:03,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:05,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:06,739 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:47:08,740 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:11,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:47:11,044 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:47:11,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:47:11,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:11,567 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:11,567 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:11,572 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:12,390 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:13,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:14,609 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:15,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:15,557 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:15,558 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:15,562 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:16,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:17,398 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:19,145 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:19,146 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:19,147 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:19,151 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:19,399 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:20,152 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:21,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:22,719 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:22,720 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:22,720 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:22,725 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:23,403 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:25,407 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:25,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:26,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:47:26,044 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:47:26,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:47:26,692 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:26,693 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:26,693 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:26,698 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:27,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:29,411 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:30,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:32,300 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:32,301 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:32,301 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:32,306 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:32,413 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:33,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:36,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:36,718 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:36,718 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:36,719 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:36,723 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:36,741 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:47:37,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:39,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:41,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:47:41,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:47:41,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:47:41,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:42,407 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:42,408 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:42,408 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:42,413 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:42,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:45,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:46,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:46,332 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:46,333 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:46,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:46,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:47,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:49,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:50,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:50,340 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:50,340 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:50,345 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:50,431 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:52,345 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:53,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:54,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:54,041 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:54,041 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:54,046 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:54,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:47:55,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:47:56,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:47:56,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:47:56,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:47:58,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:59,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:59,338 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:47:59,338 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:59,343 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:59,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:01,445 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:03,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:05,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:05,010 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:05,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:05,015 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:05,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:06,743 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:48:07,450 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:08,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:11,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:48:11,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:48:11,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:48:14,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:14,030 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:14,030 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:14,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:14,035 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:14,454 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:15,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:17,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:17,591 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:17,591 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:17,596 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:18,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:19,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:19,596 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:23,357 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:23,358 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:23,358 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:23,363 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:23,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:25,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:25,465 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:26,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:48:26,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:48:26,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:48:27,771 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:27,772 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:27,772 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:27,777 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:28,467 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:29,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:30,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:31,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:31,708 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:31,708 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:31,713 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:32,471 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:33,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:35,396 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:35,397 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:35,397 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:35,402 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:35,475 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:36,402 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:36,745 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:48:37,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:41,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:41,030 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:41,030 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:41,035 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:41,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:48:41,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:48:41,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:48:41,481 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:42,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:43,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:45,644 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:45,644 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:45,645 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:45,649 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:46,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:47,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:47,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:49,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:49,834 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:49,834 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:49,839 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:50,490 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:51,492 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:52,839 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:53,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:53,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:53,462 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:53,467 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:53,493 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:48:55,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:48:56,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:48:56,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:48:56,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:48:58,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:58,638 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:58,639 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:48:58,639 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:58,644 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:59,498 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:01,501 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:03,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:04,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:04,946 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:49:04,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:04,951 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:05,503 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:06,747 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:49:07,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:08,749 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:08,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:08,850 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:49:08,850 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:08,855 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:09,507 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:11,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:49:11,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:49:11,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:49:11,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:14,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:14,603 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:14,603 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:49:14,603 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:14,608 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:15,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:17,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:19,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:22,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:22,918 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:49:22,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:22,923 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:23,520 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:24,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:25,524 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:26,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:49:26,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:49:26,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:49:30,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:30,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:49:30,278 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:30,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:30,283 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:30,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:31,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:35,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:35,103 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:49:35,103 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:35,109 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:35,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:36,109 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:36,753 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:49:37,532 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:40,765 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:40,766 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:49:40,766 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:40,772 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:41,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:49:41,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:49:41,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:49:41,252 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:41,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:43,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:45,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:45,313 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:49:45,313 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:45,318 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:45,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:46,357 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:47,542 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:49,305 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:49,306 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:49:49,307 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:49,312 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:49,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:51,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:52,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:53,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:53,332 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:49:53,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:53,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:53,547 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:49:55,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:49:56,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:49:56,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:49:56,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:49:57,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:00,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:00,159 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:00,159 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:00,164 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:00,554 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:01,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:03,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:05,219 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:05,220 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:05,220 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:05,226 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:05,559 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:06,752 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:50:07,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:08,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:09,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:09,236 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:09,236 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:09,242 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:09,564 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:11,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:50:11,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:50:11,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:50:11,567 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:13,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:13,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:13,808 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:13,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:13,813 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:14,568 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:15,571 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:18,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:18,368 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:18,368 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:18,373 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:18,574 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:19,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:19,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:24,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:24,021 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:24,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:24,027 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:24,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:25,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:25,582 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:26,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:50:26,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:50:26,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:50:30,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:31,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:31,151 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:31,151 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:31,156 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:31,586 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:33,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:35,169 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:35,170 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:35,170 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:35,175 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:35,590 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:36,175 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:36,757 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:50:37,593 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:40,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:40,478 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:40,478 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:40,482 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:40,595 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:41,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:50:41,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:50:41,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:50:41,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:41,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:44,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:44,515 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:44,516 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:44,520 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:44,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:46,557 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:47,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:48,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:48,960 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:48,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:48,965 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:49,604 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:51,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:51,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:52,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:52,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:52,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:52,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:53,608 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:55,611 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:50:56,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:50:56,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:50:56,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:50:57,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:57,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:57,859 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:50:57,859 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:57,864 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:58,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:50:59,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:01,993 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:01,994 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:01,994 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:01,999 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:02,619 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:03,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:03,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:06,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:06,045 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:06,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:06,050 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:06,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:06,756 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:51:07,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:08,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:10,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:10,271 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:10,271 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:10,276 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:10,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:11,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:51:11,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:51:11,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:51:11,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:14,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:17,392 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:17,393 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:17,393 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:17,398 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:17,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:19,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:19,638 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:21,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:21,541 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:21,542 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:21,547 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:21,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:23,641 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:24,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:26,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:51:26,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:51:26,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:51:27,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:27,986 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:27,986 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:27,991 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:28,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:29,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:30,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:33,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:33,909 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:33,910 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:33,914 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:34,649 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:35,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:35,915 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:36,761 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:51:39,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:39,095 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:39,095 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:39,101 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:39,654 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:41,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:51:41,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:51:41,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:51:41,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:41,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:46,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:47,375 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:47,375 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:47,376 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:47,380 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:47,660 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:49,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:51,417 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:51,891 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:51,892 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:51,893 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:51,897 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:52,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:53,668 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:51:55,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:55,646 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:51:55,646 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:55,651 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:55,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:51:56,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:51:56,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:51:56,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:51:57,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:57,672 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:00,128 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:00,129 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:00,129 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:00,134 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:00,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:01,677 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:03,173 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:05,180 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:05,181 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:05,181 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:05,187 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:05,680 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:06,760 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:52:07,683 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:08,761 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:09,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:09,022 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:09,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:09,030 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:09,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:11,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:52:11,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:52:11,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:52:11,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:12,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:12,670 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:12,670 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:12,675 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:12,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:14,713 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:15,691 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:15,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:15,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:15,990 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:15,995 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:16,693 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:17,696 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:19,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:20,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:20,562 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:20,562 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:20,567 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:20,697 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:21,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:24,494 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:24,495 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:24,495 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:24,500 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:24,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:25,500 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:25,705 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:26,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:52:26,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:52:26,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:52:28,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:28,630 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:28,630 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:28,634 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:28,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:30,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:31,710 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:34,880 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:34,881 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:34,881 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:34,886 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:35,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:35,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:36,762 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:52:37,715 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:39,720 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:39,721 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:39,721 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:39,726 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:39,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:41,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:52:41,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:52:41,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:52:41,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:41,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:44,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:44,021 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:44,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:44,027 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:44,863 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:45,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:47,068 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:48,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:48,823 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:48,823 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:48,828 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:48,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:51,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:52,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:53,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:53,935 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:53,936 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:53,940 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:54,872 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:55,875 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:52:56,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:52:56,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:52:56,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:52:58,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:58,642 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:58,643 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:52:58,643 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:58,648 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:58,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:52:59,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:03,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:04,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:04,702 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:04,702 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:04,707 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:04,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:06,764 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:53:07,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:08,766 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:11,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:53:11,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:53:11,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:53:13,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:13,240 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:13,240 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:13,245 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:13,891 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:14,281 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:15,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:17,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:17,198 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:17,198 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:17,203 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:17,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:19,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:20,204 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:21,686 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:21,686 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:21,687 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:21,692 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:21,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:23,904 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:25,693 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:26,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:53:26,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:53:26,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:53:28,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:28,614 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:28,615 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:28,619 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:28,907 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:29,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:31,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:32,674 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:32,674 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:32,675 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:32,679 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:32,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:33,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:36,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:36,766 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:53:36,776 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:36,776 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:36,777 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:36,781 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:36,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:39,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:40,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:40,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:40,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:40,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:41,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:53:41,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:53:41,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:53:41,920 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:42,352 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:43,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:44,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:44,465 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:44,465 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:44,471 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:44,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:45,927 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:47,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:48,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:48,642 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:48,642 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:48,647 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:48,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:49,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:52,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:52,442 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:52,442 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:52,447 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:52,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:53,447 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:53,935 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:53:56,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:53:56,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:53:56,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:53:57,857 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:57,858 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:53:57,858 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:57,863 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:57,938 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:53:58,904 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:59,942 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:03,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:03,055 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:03,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:03,060 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:03,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:04,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:05,948 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:06,768 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:54:07,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:07,454 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:07,455 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:07,459 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:07,949 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:09,460 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:09,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:11,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:54:11,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:54:11,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:54:12,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:12,346 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:12,347 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:12,351 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:12,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:13,956 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:15,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:16,544 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:16,545 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:16,545 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:16,550 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:16,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:17,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:20,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:20,367 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:20,368 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:20,372 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:20,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:21,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:21,966 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:24,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:24,762 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:24,763 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:24,767 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:24,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:25,971 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:26,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:54:26,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:54:26,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:54:27,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:28,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:28,923 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:28,923 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:28,927 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:28,973 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:31,976 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:32,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:36,770 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:54:37,518 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:37,519 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:37,519 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:37,524 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:37,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:38,524 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:39,983 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:41,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:54:41,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:54:41,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:54:41,652 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:41,653 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:41,653 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:41,658 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:41,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:43,693 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:43,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:45,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:45,331 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:45,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:45,336 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:45,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:47,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:49,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:49,840 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:49,841 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:49,841 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:49,846 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:49,993 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:51,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:54,638 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:54,639 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:54,639 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:54,639 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:54,644 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:54,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:54:56,001 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:54:56,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:54:56,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:54:56,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:54:58,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:58,364 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:54:58,364 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:58,369 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:59,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:00,006 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:00,405 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:03,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:03,027 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:03,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:03,032 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:04,008 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:06,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:06,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:06,776 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:55:07,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:07,451 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:07,451 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:07,456 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:08,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:10,016 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:11,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:55:11,047 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:55:11,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:55:11,311 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:14,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:14,015 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:14,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:14,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:14,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:16,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:17,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:18,643 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:18,644 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:18,644 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:18,649 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:19,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:20,029 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:22,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:24,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:24,025 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:24,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:24,031 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:24,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:26,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:26,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:55:26,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:55:26,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:55:27,982 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:27,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:27,983 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:27,983 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:27,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:28,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:30,038 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:32,578 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:32,578 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:32,579 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:32,583 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:33,040 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:33,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:34,042 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:36,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:36,364 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:36,365 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:36,369 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:36,775 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:55:37,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:38,047 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:38,776 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:41,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:55:41,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:55:41,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:55:41,169 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:41,333 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:41,333 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:41,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:42,050 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:44,053 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:44,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:47,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:47,020 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:47,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:47,025 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:47,055 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:50,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:50,059 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:51,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:51,467 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:51,467 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:51,472 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:52,060 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:55:54,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:55:55,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:56,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:55:56,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:55:56,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:55:57,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:57,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:55:57,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:57,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:58,065 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:00,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:00,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:03,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:03,120 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:03,120 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:03,125 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:04,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:06,075 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:06,125 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:06,780 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:56:08,606 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:08,607 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:08,607 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:08,612 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:09,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:10,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:11,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:56:11,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:56:11,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:56:11,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:12,713 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:12,713 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:12,713 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:12,718 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:13,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:14,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:16,519 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:16,520 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:16,520 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:16,520 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:16,524 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:17,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:18,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:20,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:20,120 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:20,120 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:20,125 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:21,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:22,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:22,125 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:26,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:56:26,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:56:26,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:56:27,313 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:28,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:28,469 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:28,469 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:28,474 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:29,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:30,101 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:32,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:32,150 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:32,150 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:32,155 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:33,103 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:33,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:34,106 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:36,506 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:36,506 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:36,507 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:36,512 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:36,779 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:56:37,108 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:38,110 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:38,780 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:41,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:56:41,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:56:41,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:56:42,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:42,317 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:42,318 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:42,323 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:43,113 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:44,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:44,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:47,152 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:47,153 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:47,153 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:47,158 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:48,118 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:50,121 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:50,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:53,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:53,454 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:53,454 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:53,460 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:54,124 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:56:55,460 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:56,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:56:56,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:56:56,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:56:56,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:56:58,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:58,459 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:56:58,459 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:58,464 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:59,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:00,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:00,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:05,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:06,635 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:06,636 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:06,636 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:06,641 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:06,784 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:57:07,136 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:08,139 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:10,675 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:10,675 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:10,676 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:10,676 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:10,681 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:11,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:57:11,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:57:11,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:57:11,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:12,141 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:14,900 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:14,900 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:14,900 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:14,905 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:15,143 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:15,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:16,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:19,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:19,209 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:19,209 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:19,214 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:20,149 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:21,215 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:22,152 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:26,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:57:26,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:57:26,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:57:26,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:27,794 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:27,794 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:27,794 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:27,799 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:28,156 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:30,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:31,405 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:31,406 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:31,406 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:31,407 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:31,411 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:32,160 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:34,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:35,269 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:35,270 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:35,270 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:35,275 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:36,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:36,783 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:57:36,785 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:38,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:40,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:40,367 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:40,367 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:40,372 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:41,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:57:41,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:57:41,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:57:41,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:42,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:42,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:44,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:44,748 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:44,749 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:44,754 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:45,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:46,177 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:47,789 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:48,731 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:48,732 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:48,732 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:48,737 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:49,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:50,181 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:52,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:52,955 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:52,955 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:52,955 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:52,960 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:53,183 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:54,185 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:57:56,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:57:56,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:57:56,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:57:56,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:56,942 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:57:56,943 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:56,947 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:57,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:57:57,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:58,189 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:00,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:00,922 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:00,922 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:00,928 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:01,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:02,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:03,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:04,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:04,718 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:04,718 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:04,723 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:05,196 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:06,199 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:06,785 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:58:08,661 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:08,665 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:08,665 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:08,670 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:09,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:09,671 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:10,204 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:11,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:58:11,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:58:11,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:58:12,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:12,580 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:12,580 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:12,586 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:13,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:14,208 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:15,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:20,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:20,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:20,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:20,695 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:20,700 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:21,213 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:22,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:24,440 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:24,441 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:24,441 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:24,446 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:25,218 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:26,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:58:26,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:58:26,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:58:26,221 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:26,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:28,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:28,940 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:28,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:28,944 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:29,223 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:30,225 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:31,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:33,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:33,107 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:33,107 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:33,112 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:33,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:36,231 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:36,787 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:58:37,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:37,012 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:37,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:37,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:37,017 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:37,231 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:38,234 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:41,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:58:41,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:58:41,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:58:41,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:41,431 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:41,431 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:41,436 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:42,236 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:42,473 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:44,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:45,592 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:45,593 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:45,593 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:45,598 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:46,241 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:47,598 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:48,244 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:50,787 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:50,787 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:50,788 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:50,793 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:51,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:58:52,248 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:58:52,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:56,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:58:56,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:58:56,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:58:58,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:59,180 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:59,181 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:58:59,181 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:59,186 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:59,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:02,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:04,221 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:05,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:05,321 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:05,322 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:05,327 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:06,259 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:06,789 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:59:08,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:09,596 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:09,596 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:09,597 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:09,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:09,602 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:10,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:11,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:59:11,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:59:11,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:59:12,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:13,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:13,355 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:13,355 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:13,360 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:14,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:15,397 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:16,271 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:19,301 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:19,302 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:19,302 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:19,307 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:20,275 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:21,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:22,278 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:23,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:23,298 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:23,298 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:23,303 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:24,279 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:26,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:59:26,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:59:26,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:59:26,283 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:26,716 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:26,717 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:26,717 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:26,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:26,722 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:27,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:28,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:31,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:35,776 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:35,777 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:35,777 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:35,782 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:36,291 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:36,782 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:36,791 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 16:59:38,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:39,222 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:39,223 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:39,223 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:39,228 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:39,295 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:41,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:59:41,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:59:41,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:59:42,298 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:42,376 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:44,592 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:44,593 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:44,593 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:44,598 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:45,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:46,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:47,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:52,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:52,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:52,981 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:52,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:52,986 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:53,307 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 16:59:54,309 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 16:59:56,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:59:56,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:59:56,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:59:58,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:58,177 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 16:59:58,178 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:58,178 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:58,182 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:58,311 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:00,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:02,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:02,062 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:02,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:02,067 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:02,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:04,068 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:04,320 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:06,604 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:06,605 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:06,605 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:06,610 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:06,794 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:00:07,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:08,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:09,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:11,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:00:11,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:00:11,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:00:14,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:14,317 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:14,317 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:14,322 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:14,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:15,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:16,330 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:20,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:23,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:23,237 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:23,237 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:23,242 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:23,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:26,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:00:26,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:00:26,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:00:26,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:26,341 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:29,169 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:29,170 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:29,170 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:29,175 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:29,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:30,345 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:32,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:33,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:33,019 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:33,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:33,024 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:33,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:34,349 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:36,799 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:00:37,800 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:39,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:39,409 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:39,409 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:39,414 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:40,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:41,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:00:41,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:00:41,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:00:42,356 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:42,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:42,989 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:42,989 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:42,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:42,994 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:43,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:44,359 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:46,512 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:46,513 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:46,513 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:46,518 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:47,361 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:48,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:48,518 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:52,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:52,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:52,853 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:52,858 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:53,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:53,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:54,369 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:00:56,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:00:56,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:00:56,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:00:58,763 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:58,764 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:00:58,764 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:58,769 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:59,373 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:00:59,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:00,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:01:04,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:04,900 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:04,901 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:01:04,901 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:04,906 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:05,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:01:06,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:01:06,798 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:01:09,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:09,324 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:01:09,324 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:09,329 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:09,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:01:10,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:11,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:01:11,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:01:11,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:01:12,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:01:16,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:18,495 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:18,496 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:01:18,496 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:18,501 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:19,391 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:01:20,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:01:21,537 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:26,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:01:26,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:01:26,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:01:27,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:27,210 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:01:27,210 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:27,210 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:27,215 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:27,400 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:01:28,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:01:32,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:32,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:32,310 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:01:32,310 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:32,315 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:32,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:01:34,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:01:36,595 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:36,595 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:01:36,596 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:36,601 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:36,803 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:01:37,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:01:37,804 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:38,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:01:41,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:01:41,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:01:41,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:01:41,593 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:41,594 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:01:41,594 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:41,599 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:42,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:01:43,637 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:44,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:01:46,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:46,000 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:01:46,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:46,005 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:46,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:01:48,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:01:49,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:51,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:51,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:01:51,739 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:51,744 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:52,423 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:01:54,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:01:54,744 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:55,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:55,327 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:01:55,328 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:55,333 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:55,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:01:56,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:01:56,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:01:56,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:01:58,429 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:00,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:03,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:03,293 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:03,293 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:03,298 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:03,433 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:05,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:06,437 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:06,802 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:02:08,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:08,681 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:08,681 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:08,686 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:09,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:10,442 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:10,687 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:11,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:02:11,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:02:11,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:02:12,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:12,926 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:12,926 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:12,931 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:13,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:14,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:15,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:16,436 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:16,437 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:16,438 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:16,442 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:16,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:18,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:20,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:20,159 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:20,159 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:20,164 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:20,453 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:21,164 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:22,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:26,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:02:26,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:02:26,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:02:26,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:26,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:26,375 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:26,375 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:26,380 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:26,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:28,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:31,417 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:33,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:33,326 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:33,326 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:33,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:33,465 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:34,468 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:36,807 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:02:36,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:37,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:37,510 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:37,510 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:37,515 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:38,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:40,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:41,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:02:41,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:02:41,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:02:42,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:43,371 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:43,372 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:43,372 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:43,377 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:43,475 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:46,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:47,413 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:48,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:48,523 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:48,523 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:48,528 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:49,480 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:50,483 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:52,529 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:55,099 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:55,100 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:55,100 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:55,105 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:55,486 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:02:56,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:02:56,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:02:56,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:02:56,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:02:58,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:59,753 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:59,754 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:02:59,754 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:59,759 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:00,492 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:02,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:03,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:03,362 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:03,362 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:03,362 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:03,367 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:03,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:04,498 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:06,806 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:03:08,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:11,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:03:11,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:03:11,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:03:11,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:11,869 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:11,869 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:11,875 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:12,503 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:13,913 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:14,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:17,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:17,236 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:17,236 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:17,241 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:17,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:18,512 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:19,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:22,207 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:22,208 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:22,208 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:22,213 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:22,514 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:24,518 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:25,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:26,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:03:26,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:03:26,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:03:26,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:26,873 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:26,873 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:26,878 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:27,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:28,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:30,913 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:31,182 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:31,183 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:31,183 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:31,188 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:31,525 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:32,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:35,685 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:35,686 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:35,686 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:35,691 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:36,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:36,692 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:36,808 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:03:38,533 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:39,468 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:39,469 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:39,469 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:39,474 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:39,533 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:41,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:03:41,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:03:41,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:03:42,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:42,537 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:46,159 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:46,159 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:46,159 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:46,165 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:46,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:48,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:48,542 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:51,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:51,078 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:51,079 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:51,083 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:51,544 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:52,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:54,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:55,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:55,263 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:55,263 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:55,268 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:55,548 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:03:56,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:03:56,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:03:56,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:03:56,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:03:59,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:59,398 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:59,398 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:03:59,398 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:59,403 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:59,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:00,556 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:04,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:04,335 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:04,335 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:04,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:04,340 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:04,558 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:06,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:06,810 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:04:08,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:08,681 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:08,681 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:08,686 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:09,563 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:09,687 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:10,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:11,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:04:11,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:04:11,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:04:13,771 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:13,772 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:13,772 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:13,777 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:14,568 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:14,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:16,571 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:18,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:18,921 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:18,922 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:18,927 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:19,574 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:19,928 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:20,577 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:23,380 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:23,381 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:23,381 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:23,386 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:23,579 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:24,582 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:25,387 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:26,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:04:26,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:04:26,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:04:27,542 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:27,543 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:27,543 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:27,548 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:27,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:30,585 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:30,588 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:33,692 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:33,693 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:33,693 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:33,698 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:34,591 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:35,699 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:36,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:36,812 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:04:37,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:37,459 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:37,459 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:37,464 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:37,594 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:38,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:41,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:04:41,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:04:41,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:04:41,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:41,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:41,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:41,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:41,516 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:41,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:44,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:45,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:45,989 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:45,989 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:45,994 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:46,604 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:46,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:48,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:50,417 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:50,418 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:50,418 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:50,424 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:50,608 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:52,424 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:52,611 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:54,608 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:54,608 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:54,609 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:54,614 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:54,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:04:56,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:04:56,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:04:56,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:04:56,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:04:58,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:59,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:59,075 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:04:59,075 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:59,080 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:59,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:00,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:03,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:03,056 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:03,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:03,061 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:03,625 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:04,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:04,627 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:06,818 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:05:07,920 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:07,921 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:07,921 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:07,926 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:08,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:09,927 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:10,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:11,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:05:11,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:05:11,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:05:13,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:13,287 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:13,287 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:13,292 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:13,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:14,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:15,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:17,611 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:17,611 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:17,612 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:17,616 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:17,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:20,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:20,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:23,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:23,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:23,853 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:23,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:24,647 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:25,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:26,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:05:26,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:05:26,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:05:26,651 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:27,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:27,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:27,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:27,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:28,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:30,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:31,660 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:31,661 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:31,661 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:31,661 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:31,666 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:32,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:34,660 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:35,589 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:35,589 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:35,590 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:35,595 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:35,661 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:36,817 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:05:36,818 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:38,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:41,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:05:41,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:05:41,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:05:41,540 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:41,541 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:41,541 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:41,546 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:41,666 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:42,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:44,670 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:47,155 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:47,156 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:47,156 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:47,161 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:47,672 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:48,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:48,674 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:50,860 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:50,861 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:50,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:50,866 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:51,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:52,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:53,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:54,666 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:54,667 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:05:54,667 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:54,673 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:54,680 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:05:56,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:05:56,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:05:56,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:05:56,683 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:05:59,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:00,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:00,045 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:00,045 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:00,050 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:00,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:02,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:04,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:04,690 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:04,690 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:04,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:04,695 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:05,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:06,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:06,822 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:06:09,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:11,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:06:11,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:06:11,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:06:11,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:11,292 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:11,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:11,297 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:11,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:12,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:15,333 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:16,200 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:16,200 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:16,201 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:16,206 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:16,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:18,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:20,352 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:20,353 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:20,354 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:20,354 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:20,359 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:20,708 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:22,711 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:24,630 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:24,631 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:24,631 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:24,636 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:24,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:25,637 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:26,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:06:26,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:06:26,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:06:26,716 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:28,564 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:28,564 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:28,565 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:28,569 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:28,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:30,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:31,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:31,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:31,932 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:31,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:31,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:32,722 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:34,726 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:35,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:35,747 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:35,747 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:35,752 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:36,727 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:36,752 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:36,821 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:06:38,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:39,658 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:39,659 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:39,659 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:39,664 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:39,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:41,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:06:41,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:06:41,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:06:42,336 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:42,734 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:43,676 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:43,677 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:43,677 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:43,682 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:43,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:46,739 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:47,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:48,327 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:48,328 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:48,328 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:48,333 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:48,740 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:50,743 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:52,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:52,557 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:52,557 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:52,563 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:52,744 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:53,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:54,747 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:06:56,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:06:56,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:06:56,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:06:58,116 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:58,117 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:06:58,117 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:58,122 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:58,750 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:06:59,157 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:00,754 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:03,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:03,811 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:03,811 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:03,816 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:04,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:04,817 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:06,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:06,826 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:07:07,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:07,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:07,697 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:07,702 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:07,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:10,702 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:10,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:11,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:07:11,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:07:11,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:07:11,395 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:11,395 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:11,395 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:11,400 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:11,765 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:12,767 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:15,442 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:15,443 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:15,443 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:15,448 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:15,769 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:16,449 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:16,771 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:19,432 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:19,433 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:19,433 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:19,438 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:19,774 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:20,777 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:22,439 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:23,000 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:23,000 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:23,001 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:23,005 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:23,779 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:24,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:26,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:07:26,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:07:26,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:07:28,028 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:28,029 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:28,029 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:28,029 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:28,034 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:28,785 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:30,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:33,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:36,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:36,318 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:36,318 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:36,324 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:36,791 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:36,825 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:07:38,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:38,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:40,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:40,690 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:40,690 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:40,695 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:40,796 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:41,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:07:41,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:07:41,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:07:42,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:44,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:44,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:44,258 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:44,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:44,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:44,800 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:46,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:49,301 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:52,662 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:52,663 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:52,664 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:52,669 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:52,804 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:54,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:54,808 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:07:56,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:07:56,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:07:56,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:07:58,740 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:58,740 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:07:58,740 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:58,746 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:58,810 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:07:59,781 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:00,814 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:02,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:02,096 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:02,097 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:02,101 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:02,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:04,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:05,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:06,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:06,483 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:06,483 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:06,488 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:06,820 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:06,830 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:08:08,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:10,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:10,305 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:10,306 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:10,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:10,310 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:10,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:11,051 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:08:11,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:08:11,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:08:12,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:14,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:14,063 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:14,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:14,068 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:14,829 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:16,105 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:16,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:21,105 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:21,626 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:21,627 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:21,627 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:21,632 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:21,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:22,838 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:26,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:08:26,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:08:26,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:08:26,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:26,275 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:26,275 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:26,275 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:26,280 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:26,841 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:28,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:31,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:31,070 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:31,070 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:31,076 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:31,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:32,113 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:32,848 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:34,655 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:34,655 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:34,656 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:34,661 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:34,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:36,829 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:08:36,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:37,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:39,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:39,187 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:39,187 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:39,192 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:39,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:40,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:41,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:08:41,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:08:41,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:08:43,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:43,280 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:43,280 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:43,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:43,285 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:43,859 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:44,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:47,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:47,031 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:47,031 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:47,036 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:47,863 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:48,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:49,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:51,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:51,131 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:51,131 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:51,136 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:51,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:52,869 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:54,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:55,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:55,952 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:55,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:55,957 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:56,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:08:56,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:08:56,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:08:56,872 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:08:58,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:08:59,304 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:59,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:59,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:08:59,697 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:59,702 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:59,875 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:00,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:03,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:03,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:03,296 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:03,301 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:03,879 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:04,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:05,301 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:06,835 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:09:07,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:07,197 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:07,198 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:07,203 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:07,883 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:08,886 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:11,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:09:11,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:09:11,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:09:11,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:11,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:11,297 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:11,297 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:11,302 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:11,888 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:12,890 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:16,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:17,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:17,064 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:17,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:17,069 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:17,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:18,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:22,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:22,062 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:22,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:22,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:22,068 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:22,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:24,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:26,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:09:26,052 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:09:26,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:09:26,424 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:26,424 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:26,424 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:26,429 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:26,904 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:27,465 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:28,907 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:30,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:30,055 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:30,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:30,061 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:30,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:32,912 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:33,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:34,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:34,327 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:34,327 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:34,332 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:34,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:36,834 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:09:36,916 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:38,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:38,307 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:38,308 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:38,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:38,313 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:38,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:40,920 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:41,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:09:41,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:09:41,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:09:44,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:46,732 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:46,733 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:46,733 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:46,738 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:46,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:48,927 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:49,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:51,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:51,591 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:51,591 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:51,596 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:51,928 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:52,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:09:55,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:56,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:09:56,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:09:56,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:09:56,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:56,585 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:09:56,585 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:56,590 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:56,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:09:58,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:00,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:02,555 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:02,556 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:02,557 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:02,562 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:02,940 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:04,944 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:06,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:06,836 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:10:07,105 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:07,105 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:07,106 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:07,110 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:07,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:08,948 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:11,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:10:11,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:10:11,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:10:12,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:15,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:15,260 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:15,260 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:15,265 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:15,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:16,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:18,301 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:19,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:19,210 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:19,210 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:19,215 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:19,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:20,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:24,216 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:24,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:24,820 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:24,820 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:24,826 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:24,963 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:26,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:10:26,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:10:26,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:10:26,966 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:29,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:31,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:31,854 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:31,854 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:31,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:31,969 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:32,972 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:34,897 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:36,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:36,003 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:36,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:36,008 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:36,838 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:10:36,974 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:38,977 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:40,839 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:41,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:10:41,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:10:41,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:10:42,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:42,980 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:42,981 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:42,986 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:43,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:44,983 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:46,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:47,499 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:47,500 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:47,500 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:47,505 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:47,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:48,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:51,506 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:51,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:51,749 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:51,749 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:51,754 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:51,989 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:52,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:10:56,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:10:56,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:10:56,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:10:56,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:56,937 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:10:56,937 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:56,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:56,942 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:56,994 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:10:58,997 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:01,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:01,497 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:01,497 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:01,502 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:02,000 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:02,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:03,003 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:05,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:05,746 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:05,746 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:05,751 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:06,005 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:06,840 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:11:07,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:07,841 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:10,420 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:10,421 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:10,421 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:10,426 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:11,010 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:11,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:11:11,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:11:11,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:11:13,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:13,347 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:15,714 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:15,714 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:15,715 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:15,720 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:16,014 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:17,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:18,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:19,646 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:19,647 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:19,647 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:19,652 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:20,020 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:21,022 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:24,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:24,386 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:24,386 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:24,387 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:24,391 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:25,025 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:26,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:11:26,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:11:26,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:11:27,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:28,333 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:28,334 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:28,334 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:28,339 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:29,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:30,377 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:31,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:32,743 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:32,744 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:32,744 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:32,749 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:33,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:35,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:35,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:36,842 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:11:37,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:37,322 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:37,322 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:37,327 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:38,037 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:39,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:41,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:11:41,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:11:41,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:11:41,126 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:41,349 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:41,349 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:41,350 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:41,355 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:42,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:43,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:46,356 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:49,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:49,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:49,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:49,473 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:50,049 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:51,051 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:51,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:53,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:53,705 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:53,705 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:53,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:54,053 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:11:55,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:11:56,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:11:56,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:11:56,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:11:57,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:58,876 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:58,876 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:11:58,877 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:58,882 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:59,058 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:01,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:02,809 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:02,810 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:02,810 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:02,811 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:02,815 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:03,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:05,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:06,844 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:12:07,846 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:10,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:10,321 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:10,321 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:10,326 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:11,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:12:11,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:12:11,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:12:11,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:13,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:13,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:13,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:13,862 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:13,862 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:13,867 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:14,074 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:15,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:17,962 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:17,963 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:17,963 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:17,968 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:18,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:18,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:19,082 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:22,804 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:22,805 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:22,806 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:22,811 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:23,084 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:24,811 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:25,088 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:26,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:12:26,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:12:26,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:12:28,394 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:28,394 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:28,395 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:28,400 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:29,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:30,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:31,093 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:32,240 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:32,241 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:32,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:32,246 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:33,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:35,098 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:36,091 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:36,092 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:36,092 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:36,092 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:36,098 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:36,099 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:36,850 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:12:39,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:39,799 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:39,800 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:39,801 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:39,805 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:40,101 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:41,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:12:41,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:12:41,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:12:41,103 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:41,349 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:43,584 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:43,585 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:43,585 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:43,590 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:44,105 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:45,108 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:46,630 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:47,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:47,577 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:47,577 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:47,581 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:48,109 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:49,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:52,551 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:52,552 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:52,552 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:52,552 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:52,557 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:53,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:55,117 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:12:56,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:12:56,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:12:56,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:12:56,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:56,351 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:12:56,351 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:56,356 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:57,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:12:58,357 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:59,122 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:00,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:00,130 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:00,130 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:00,136 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:01,124 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:03,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:04,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:04,203 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:04,203 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:04,204 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:04,209 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:05,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:06,849 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:13:07,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:09,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:09,309 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:09,309 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:09,310 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:09,315 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:10,133 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:11,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:13:11,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:13:11,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:13:11,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:13,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:13,067 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:13,067 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:13,072 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:13,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:15,110 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:15,138 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:18,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:18,265 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:18,266 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:18,270 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:19,142 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:20,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:21,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:22,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:22,015 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:22,016 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:22,021 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:22,146 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:23,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:26,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:26,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:13:26,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:13:26,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:13:30,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:30,027 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:30,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:30,032 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:30,153 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:31,070 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:31,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:33,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:33,619 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:33,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:33,624 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:34,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:35,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:36,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:36,854 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:13:37,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:37,682 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:37,682 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:37,687 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:38,161 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:39,164 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:41,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:13:41,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:13:41,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:13:41,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:41,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:41,523 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:41,529 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:42,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:42,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:43,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:45,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:45,950 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:45,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:45,956 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:46,169 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:47,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:47,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:49,500 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:49,501 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:49,501 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:49,506 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:50,174 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:51,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:53,507 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:53,890 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:53,891 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:53,891 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:53,896 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:54,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:13:55,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:13:56,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:13:56,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:13:56,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:13:58,599 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:58,600 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:13:58,600 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:58,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:58,605 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:59,181 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:01,185 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:02,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:02,356 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:02,356 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:02,361 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:03,186 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:04,362 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:05,189 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:06,853 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:14:07,345 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:07,346 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:07,346 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:07,351 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:08,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:09,193 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:10,352 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:11,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:14:11,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:14:11,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:14:11,502 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:11,502 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:11,503 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:11,508 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:12,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:13,198 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:15,546 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:15,837 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:15,837 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:15,838 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:15,843 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:16,199 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:17,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:20,253 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:20,254 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:20,255 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:20,259 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:21,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:21,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:23,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:24,660 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:24,661 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:24,661 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:24,666 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:25,210 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:26,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:14:26,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:14:26,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:14:26,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:27,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:28,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:28,651 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:28,651 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:28,656 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:29,215 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:31,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:31,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:36,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:36,309 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:36,310 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:36,314 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:36,858 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:14:36,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:37,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:39,223 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:41,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:14:41,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:14:41,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:14:42,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:46,175 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:46,176 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:46,176 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:46,182 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:46,227 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:48,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:49,231 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:51,778 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:51,779 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:51,779 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:51,784 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:52,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:53,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:14:53,784 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:56,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:14:56,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:14:56,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:14:57,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:57,005 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:14:57,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:57,010 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:57,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:14:59,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:59,241 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:01,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:01,606 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:01,606 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:01,611 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:02,244 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:03,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:04,611 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:05,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:05,970 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:05,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:05,975 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:06,248 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:06,857 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:15:07,250 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:09,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:11,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:15:11,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:15:11,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:15:11,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:11,343 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:11,343 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:11,348 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:12,253 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:13,256 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:14,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:14,950 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:14,950 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:14,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:14,955 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:15,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:17,260 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:18,777 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:18,778 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:18,778 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:18,783 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:19,263 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:20,784 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:21,266 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:22,514 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:22,515 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:22,515 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:22,520 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:23,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:25,270 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:26,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:15:26,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:15:26,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:15:26,347 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:26,713 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:26,713 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:26,714 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:26,719 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:27,272 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:29,275 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:30,709 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:30,709 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:30,710 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:30,715 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:31,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:31,716 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:33,279 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:34,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:34,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:34,438 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:34,443 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:35,280 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:36,862 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:15:36,864 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:37,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:41,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:41,050 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:41,050 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:41,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:15:41,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:15:41,055 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:41,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:15:41,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:42,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:43,287 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:44,883 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:44,883 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:44,884 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:44,889 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:45,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:47,291 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:47,926 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:50,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:50,171 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:50,172 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:50,177 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:50,293 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:15:51,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:15:53,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:56,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:15:56,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:15:56,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:15:58,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:59,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:59,117 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:15:59,117 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:59,122 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:59,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:01,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:03,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:03,947 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:03,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:03,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:03,952 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:04,307 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:05,309 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:06,861 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:16:09,863 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:11,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:16:11,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:16:11,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:16:12,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:12,349 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:12,349 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:12,354 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:13,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:15,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:15,390 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:16,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:16,739 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:16,739 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:16,745 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:17,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:19,321 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:20,746 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:21,476 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:21,477 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:21,477 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:21,482 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:22,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:23,325 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:25,519 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:25,520 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:25,520 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:25,525 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:26,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:16:26,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:16:26,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:16:26,328 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:26,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:27,330 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:30,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:30,516 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:30,516 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:30,521 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:31,333 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:31,558 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:33,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:36,558 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:36,863 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:16:37,412 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:37,413 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:37,414 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:37,419 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:38,339 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:39,341 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:41,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:16:41,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:16:41,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:16:42,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:42,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:42,703 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:42,704 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:42,709 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:43,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:45,346 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:46,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:46,869 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:46,870 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:46,874 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:47,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:47,875 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:49,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:52,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:52,726 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:52,726 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:52,731 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:53,352 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:53,731 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:55,355 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:16:56,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:16:56,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:16:56,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:16:57,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:57,286 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:16:57,286 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:57,291 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:57,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:16:59,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:59,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:02,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:02,025 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:02,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:02,030 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:02,363 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:03,365 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:05,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:06,865 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:17:07,181 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:07,182 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:07,182 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:07,187 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:07,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:09,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:10,188 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:10,976 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:10,977 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:10,977 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:10,982 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:11,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:17:11,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:17:11,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:17:11,372 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:13,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:15,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:15,023 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:15,024 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:15,029 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:15,376 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:16,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:17,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:20,175 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:20,176 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:20,176 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:20,181 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:20,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:21,182 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:21,384 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:23,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:23,757 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:23,758 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:23,763 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:24,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:25,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:26,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:17:26,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:17:26,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:17:26,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:28,947 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:28,948 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:28,948 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:28,953 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:29,390 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:31,393 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:31,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:33,973 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:33,974 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:33,975 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:33,980 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:34,394 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:35,397 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:36,867 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:17:37,839 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:37,840 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:37,840 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:37,840 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:37,845 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:38,398 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:39,401 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:41,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:17:41,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:17:41,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:17:43,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:43,154 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:43,154 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:43,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:43,159 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:43,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:45,407 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:46,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:46,972 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:46,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:46,977 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:47,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:48,977 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:49,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:53,978 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:55,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:55,317 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:55,317 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:55,323 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:55,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:17:56,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:17:56,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:17:56,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:17:57,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:17:59,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:59,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:59,853 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:17:59,853 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:59,858 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:00,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:01,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:04,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:06,872 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:18:08,077 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:08,078 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:08,078 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:08,083 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:08,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:09,428 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:10,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:11,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:18:11,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:18:11,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:18:12,871 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:12,872 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:12,872 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:12,877 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:13,430 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:15,433 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:15,913 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:16,688 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:16,689 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:16,690 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:16,695 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:17,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:19,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:20,759 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:20,760 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:20,760 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:20,765 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:21,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:21,765 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:23,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:26,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:18:26,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:18:26,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:18:26,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:26,478 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:26,478 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:26,483 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:27,446 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:27,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:29,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:30,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:30,497 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:30,498 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:30,502 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:31,450 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:33,453 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:33,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:36,871 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:18:37,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:37,103 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:37,104 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:37,109 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:37,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:39,109 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:39,458 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:41,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:18:41,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:18:41,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:18:41,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:41,382 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:41,382 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:41,387 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:41,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:43,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:44,425 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:45,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:45,620 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:45,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:45,626 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:46,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:47,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:49,626 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:49,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:49,704 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:49,704 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:49,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:50,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:51,469 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:54,074 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:54,075 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:54,075 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:54,080 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:54,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:18:55,081 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:56,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:18:56,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:18:56,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:18:57,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:18:58,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:58,308 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:18:58,308 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:58,313 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:58,475 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:00,349 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:01,479 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:03,032 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:03,033 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:03,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:03,038 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:03,481 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:05,483 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:06,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:06,595 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:06,596 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:06,597 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:06,601 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:06,876 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:19:07,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:09,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:11,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:19:11,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:19:11,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:19:11,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:15,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:15,053 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:15,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:15,059 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:15,491 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:17,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:17,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:18,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:18,873 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:18,874 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:18,879 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:19,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:21,499 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:22,879 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:22,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:22,967 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:22,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:22,973 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:23,501 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:25,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:26,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:19:26,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:19:26,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:19:28,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:28,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:28,438 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:28,438 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:28,443 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:28,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:31,510 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:32,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:32,516 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:32,516 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:32,521 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:33,511 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:33,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:35,514 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:36,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:36,822 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:36,822 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:36,828 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:36,876 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:19:37,515 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:38,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:39,518 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:40,595 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:40,596 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:40,596 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:40,601 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:41,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:19:41,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:19:41,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:19:41,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:43,522 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:44,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:44,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:44,995 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:44,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:45,000 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:45,524 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:47,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:50,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:50,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:50,271 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:50,272 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:50,277 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:50,529 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:53,532 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:19:55,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:56,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:19:56,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:19:56,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:19:56,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:56,748 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:19:56,748 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:56,753 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:57,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:19:59,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:00,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:05,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:05,207 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:05,207 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:05,212 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:05,542 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:06,212 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:06,881 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:20:07,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:08,800 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:08,801 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:08,801 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:08,806 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:09,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:11,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:20:11,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:20:11,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:20:11,308 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:11,549 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:12,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:12,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:12,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:12,997 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:13,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:15,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:17,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:17,161 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:17,162 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:17,162 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:17,167 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:17,555 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:19,559 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:22,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:22,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:22,940 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:22,940 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:22,945 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:23,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:25,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:26,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:20:26,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:20:26,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:20:27,071 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:27,071 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:27,072 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:27,077 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:27,566 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:28,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:29,569 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:33,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:35,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:35,578 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:35,578 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:35,583 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:36,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:36,880 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:20:37,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:38,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:39,693 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:39,694 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:39,694 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:39,699 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:40,577 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:41,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:20:41,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:20:41,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:20:41,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:43,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:43,642 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:43,642 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:43,647 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:44,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:44,686 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:45,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:49,686 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:52,138 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:52,139 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:52,139 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:52,144 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:52,588 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:55,145 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:55,591 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:20:56,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:20:56,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:20:56,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:20:57,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:57,422 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:20:57,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:57,427 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:57,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:20:59,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:00,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:02,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:02,293 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:02,293 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:02,298 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:02,598 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:05,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:06,299 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:06,885 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:21:07,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:07,167 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:07,167 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:07,172 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:07,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:09,606 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:11,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:21:11,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:21:11,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:21:11,627 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:11,627 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:11,628 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:11,628 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:11,632 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:12,608 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:13,610 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:16,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:17,229 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:17,230 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:17,230 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:17,235 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:17,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:19,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:22,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:25,625 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:25,626 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:25,626 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:25,631 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:26,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:21:26,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:21:26,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:21:26,621 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:27,356 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:27,624 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:30,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:30,326 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:30,326 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:30,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:30,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:32,370 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:33,629 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:34,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:34,831 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:34,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:34,836 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:35,631 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:36,884 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:21:37,634 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:37,885 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:40,760 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:40,761 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:40,761 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:40,766 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:41,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:21:41,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:21:41,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:21:41,636 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:43,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:43,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:44,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:44,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:44,697 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:44,702 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:45,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:47,641 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:48,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:48,313 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:48,313 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:48,313 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:48,318 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:48,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:51,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:53,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:53,500 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:53,501 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:53,501 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:53,506 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:53,647 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:55,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:21:56,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:21:56,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:21:56,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:21:57,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:57,117 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:21:57,118 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:57,122 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:57,651 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:21:59,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:59,654 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:02,068 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:02,068 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:02,069 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:02,074 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:02,657 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:03,660 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:05,074 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:06,886 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:22:07,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:07,935 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:07,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:07,941 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:08,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:09,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:10,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:11,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:22:11,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:22:11,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:22:12,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:12,306 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:12,306 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:12,311 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:12,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:15,670 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:16,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:17,787 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:17,788 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:17,789 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:17,794 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:18,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:19,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:21,794 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:23,147 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:23,148 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:23,148 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:23,152 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:23,678 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:25,682 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:26,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:22:26,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:22:26,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:22:27,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:27,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:27,815 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:27,815 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:27,820 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:28,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:29,686 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:32,420 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:32,421 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:32,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:32,422 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:32,427 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:32,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:35,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:36,888 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:22:37,890 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:38,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:38,618 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:38,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:38,623 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:38,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:41,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:22:41,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:22:41,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:22:41,698 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:43,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:43,191 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:43,191 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:43,191 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:43,196 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:43,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:45,702 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:48,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:49,427 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:49,428 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:49,428 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:49,433 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:49,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:51,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:53,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:53,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:53,560 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:53,561 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:53,565 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:53,708 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:55,711 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:22:56,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:22:56,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:22:56,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:22:57,779 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:57,780 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:22:57,780 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:57,785 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:58,713 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:22:58,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:59,716 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:01,586 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:01,587 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:01,587 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:01,593 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:01,719 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:03,722 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:04,593 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:05,498 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:05,499 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:05,499 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:05,504 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:05,723 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:06,890 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:23:07,726 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:09,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:09,515 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:09,516 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:09,521 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:09,727 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:10,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:11,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:23:11,057 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:23:11,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:23:11,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:13,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:13,516 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:13,516 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:13,521 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:13,731 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:15,558 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:15,734 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:17,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:17,177 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:17,177 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:17,182 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:17,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:19,740 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:21,182 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:21,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:21,601 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:21,601 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:21,606 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:21,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:23,744 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:26,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:23:26,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:23:26,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:23:26,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:27,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:27,286 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:27,287 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:27,292 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:27,747 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:29,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:31,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:34,888 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:34,892 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:34,892 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:34,897 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:35,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:36,892 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:23:36,894 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:37,755 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:38,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:38,918 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:38,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:38,923 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:39,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:41,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:23:41,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:23:41,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:23:41,759 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:42,371 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:43,222 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:43,223 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:43,223 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:43,228 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:43,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:45,763 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:48,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:48,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:48,887 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:48,888 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:48,893 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:49,766 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:51,769 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:53,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:53,013 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:23:53,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:53,022 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:53,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:23:54,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:55,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:23:56,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:23:56,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:23:56,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:23:59,303 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:02,634 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:02,635 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:02,635 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:02,640 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:02,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:04,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:05,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:05,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:05,947 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:05,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:05,952 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:06,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:06,894 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:24:07,785 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:09,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:10,864 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:10,865 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:10,865 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:10,871 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:11,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:24:11,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:24:11,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:24:11,787 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:13,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:14,816 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:14,817 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:14,817 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:14,822 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:15,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:15,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:17,795 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:18,978 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:18,979 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:18,979 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:18,984 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:19,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:20,984 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:21,800 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:22,503 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:22,504 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:22,504 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:22,509 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:22,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:25,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:26,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:24:26,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:24:26,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:24:26,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:26,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:26,830 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:26,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:26,835 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:27,806 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:29,810 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:30,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:30,854 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:30,854 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:30,858 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:31,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:31,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:33,814 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:36,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:36,900 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:24:36,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:36,954 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:36,954 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:36,959 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:37,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:39,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:40,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:40,731 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:40,731 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:40,736 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:40,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:41,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:24:41,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:24:41,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:24:42,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:43,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:44,688 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:44,689 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:44,689 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:44,694 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:44,824 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:47,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:47,827 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:48,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:48,557 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:48,557 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:48,562 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:48,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:51,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:52,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:52,699 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:52,699 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:52,704 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:52,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:53,705 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:55,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:24:56,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:24:56,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:24:56,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:24:56,663 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:56,663 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:24:56,663 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:56,668 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:56,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:24:58,706 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:59,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:00,784 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:00,785 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:00,785 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:00,791 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:00,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:03,791 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:03,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:04,768 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:04,769 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:04,769 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:04,774 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:04,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:06,899 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:25:07,848 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:08,900 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:09,173 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:09,174 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:09,174 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:09,179 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:09,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:11,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:25:11,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:25:11,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:25:11,852 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:13,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:13,826 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:13,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:13,831 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:13,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:14,870 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:15,856 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:17,755 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:17,756 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:17,756 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:17,761 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:17,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:19,862 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:20,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:24,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:24,039 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:24,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:24,044 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:24,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:25,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:26,044 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:26,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:25:26,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:25:26,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:25:29,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:29,011 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:29,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:29,016 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:29,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:31,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:31,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:32,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:32,959 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:32,959 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:32,964 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:33,874 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:35,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:36,904 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:25:36,905 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:39,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:39,577 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:39,577 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:39,583 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:39,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:41,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:25:41,058 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:25:41,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:25:41,883 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:42,328 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:45,384 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:45,385 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:45,385 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:45,390 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:45,885 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:47,426 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:47,888 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:49,148 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:49,149 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:49,149 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:49,154 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:49,889 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:51,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:53,155 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:53,679 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:53,680 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:53,680 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:53,685 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:53,894 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:55,897 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:25:56,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:25:56,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:25:56,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:25:57,372 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:57,373 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:25:57,373 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:57,377 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:57,898 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:25:58,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:59,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:02,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:02,316 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:02,316 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:02,321 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:02,904 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:03,906 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:04,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:06,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:06,354 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:06,354 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:06,359 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:06,903 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:26:06,908 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:07,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:09,905 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:10,145 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:10,145 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:10,146 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:10,151 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:10,912 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:11,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:26:11,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:26:11,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:26:11,915 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:15,350 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:18,760 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:18,760 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:18,760 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:18,765 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:18,920 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:20,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:21,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:23,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:23,575 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:23,576 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:23,581 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:23,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:25,928 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:26,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:26:26,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:26:26,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:26:26,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:27,391 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:27,392 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:27,392 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:27,397 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:27,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:29,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:31,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:32,445 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:32,446 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:32,446 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:32,451 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:32,934 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:33,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:36,452 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:36,908 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:26:37,985 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:37,986 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:37,986 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:37,991 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:38,940 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:39,942 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:41,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:26:41,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:26:41,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:26:41,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:41,824 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:41,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:41,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:41,829 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:41,944 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:43,947 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:46,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:47,791 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:47,791 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:47,792 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:47,796 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:47,949 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:49,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:52,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:53,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:53,266 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:53,266 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:53,271 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:53,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:55,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:26:56,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:26:56,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:26:56,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:26:57,995 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:57,996 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:26:57,996 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:57,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:58,001 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:58,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:26:59,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:03,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:04,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:04,683 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:04,683 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:04,688 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:04,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:06,907 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:27:07,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:08,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:09,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:09,224 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:09,224 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:09,229 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:09,968 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:11,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:27:11,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:27:11,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:27:11,971 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:14,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:15,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:15,245 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:15,245 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:15,250 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:15,974 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:17,977 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:19,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:20,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:20,095 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:20,095 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:20,100 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:20,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:21,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:24,341 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:24,342 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:24,342 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:24,342 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:24,347 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:24,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:25,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:26,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:27:26,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:27:26,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:27:30,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:30,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:30,717 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:30,718 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:30,723 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:30,990 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:33,994 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:34,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:34,273 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:34,273 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:34,278 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:34,995 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:35,997 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:36,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:36,915 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:27:38,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:38,814 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:38,814 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:38,819 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:38,999 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:41,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:27:41,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:27:41,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:27:41,345 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:42,002 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:43,364 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:43,364 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:43,365 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:43,369 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:44,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:46,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:46,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:47,732 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:47,733 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:47,733 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:47,738 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:48,008 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:50,011 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:51,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:52,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:52,435 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:27:52,435 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:52,440 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:53,013 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:27:54,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:27:56,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:27:56,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:27:56,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:27:57,349 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:01,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:01,768 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:01,768 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:01,773 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:02,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:28:02,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:04,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:28:06,911 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:28:07,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:07,820 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:07,821 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:07,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:07,826 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:08,026 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:28:10,029 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:28:11,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:28:11,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:28:11,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:28:12,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:12,981 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:12,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:12,982 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:12,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:13,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:28:16,034 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:28:17,020 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:17,021 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:17,021 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:17,026 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:17,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:28:18,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:20,040 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:28:21,926 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:21,927 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:21,927 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:21,932 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:22,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:28:23,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:24,044 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:28:26,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:28:26,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:28:26,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:28:29,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:30,560 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:30,560 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:30,561 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:30,565 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:31,049 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:28:32,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:28:34,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:35,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:35,340 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:35,340 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:35,345 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:36,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:28:36,914 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:28:38,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:28:39,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:39,322 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:39,322 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:39,327 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:40,058 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:28:40,327 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:41,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:28:41,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:28:41,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:28:42,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:28:44,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:44,242 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:44,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:44,247 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:45,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:28:46,065 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:28:46,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:51,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:52,911 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:52,912 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:52,912 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:52,917 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:53,069 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:28:56,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:28:56,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:28:56,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:28:56,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:28:56,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:59,217 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:59,218 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:28:59,218 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:59,223 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:00,075 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:02,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:02,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:02,979 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:02,980 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:02,980 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:02,985 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:03,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:06,084 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:06,915 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:29:07,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:07,161 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:07,161 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:07,166 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:08,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:08,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:10,088 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:11,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:29:11,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:29:11,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:29:12,432 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:12,433 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:12,433 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:12,438 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:13,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:13,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:14,092 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:16,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:16,288 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:16,288 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:16,293 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:17,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:18,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:19,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:21,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:21,177 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:21,178 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:21,183 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:22,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:24,102 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:25,156 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:25,157 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:25,158 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:25,158 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:25,163 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:26,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:29:26,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:29:26,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:29:26,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:28,106 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:29,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:29,338 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:29,338 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:29,343 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:30,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:30,382 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:32,110 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:33,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:33,209 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:33,209 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:33,214 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:34,111 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:36,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:36,214 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:36,918 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:29:37,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:37,053 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:37,053 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:37,058 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:37,115 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:40,118 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:40,840 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:40,841 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:40,841 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:40,846 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:41,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:29:41,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:29:41,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:29:41,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:41,306 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:44,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:44,612 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:44,613 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:44,613 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:44,618 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:45,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:46,126 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:46,654 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:48,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:48,237 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:48,237 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:48,242 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:49,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:50,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:52,243 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:53,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:53,024 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:53,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:53,029 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:53,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:29:56,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:29:56,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:29:56,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:29:56,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:29:57,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:58,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:58,649 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:29:58,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:58,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:59,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:00,139 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:02,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:02,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:02,831 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:02,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:02,836 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:03,142 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:06,146 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:06,920 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:30:07,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:07,017 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:07,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:07,023 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:07,146 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:08,023 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:10,150 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:11,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:30:11,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:30:11,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:30:11,557 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:11,558 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:11,558 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:11,563 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:12,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:13,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:14,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:17,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:17,431 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:17,431 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:17,436 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:18,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:19,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:20,161 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:22,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:22,464 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:22,464 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:22,469 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:23,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:24,165 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:24,469 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:26,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:30:26,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:30:26,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:30:26,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:26,802 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:26,803 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:26,807 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:27,167 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:29,846 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:30,171 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:31,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:31,318 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:31,318 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:31,323 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:32,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:34,175 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:35,324 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:36,922 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:30:39,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:39,935 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:39,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:39,940 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:40,182 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:40,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:41,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:30:41,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:30:41,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:30:42,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:46,301 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:46,764 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:46,765 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:46,765 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:46,770 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:47,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:50,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:50,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:50,386 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:50,386 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:50,391 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:51,200 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:51,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:30:51,391 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:56,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:30:56,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:30:56,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:30:56,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:56,682 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:30:56,682 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:56,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:56,687 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:57,204 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:30:58,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:01,464 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:01,465 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:01,465 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:01,470 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:02,212 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:02,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:03,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:05,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:05,445 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:05,445 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:05,450 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:06,218 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:06,927 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:31:07,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:07,928 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:11,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:31:11,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:31:11,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:31:13,356 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:14,798 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:14,799 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:14,799 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:14,804 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:15,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:17,234 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:18,842 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:19,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:19,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:19,699 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:19,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:20,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:21,240 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:24,431 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:24,432 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:24,432 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:24,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:24,437 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:25,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:25,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:26,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:31:26,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:31:26,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:31:30,248 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:30,248 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:30,249 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:30,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:30,254 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:30,254 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:31,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:34,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:34,542 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:34,542 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:34,547 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:35,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:35,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:35,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:36,926 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:31:38,662 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:38,663 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:38,663 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:38,668 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:39,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:39,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:40,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:41,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:31:41,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:31:41,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:31:44,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:44,684 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:44,684 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:44,689 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:45,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:45,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:45,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:49,518 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:49,519 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:49,519 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:49,525 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:50,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:51,285 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:51,525 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:53,649 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:53,650 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:53,650 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:53,655 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:54,287 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:55,290 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:31:56,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:31:56,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:31:56,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:31:57,348 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:57,780 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:57,780 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:31:57,780 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:57,785 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:58,293 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:31:59,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:02,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:02,168 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:02,168 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:02,173 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:02,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:03,173 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:03,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:06,931 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:32:07,451 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:07,452 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:07,452 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:07,457 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:08,308 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:08,457 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:09,310 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:11,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:32:11,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:32:11,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:32:13,174 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:13,174 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:13,175 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:13,179 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:13,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:14,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:15,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:16,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:16,869 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:16,869 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:16,874 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:17,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:17,322 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:19,875 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:20,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:20,798 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:20,798 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:20,803 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:21,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:21,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:24,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:24,811 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:24,811 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:24,816 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:25,333 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:25,333 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:25,816 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:26,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:32:26,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:32:26,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:32:28,624 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:28,625 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:28,625 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:28,630 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:29,338 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:29,339 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:31,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:32,548 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:32,548 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:32,549 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:32,553 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:33,344 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:33,344 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:36,930 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:32:36,931 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:37,404 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:37,404 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:37,405 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:37,409 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:38,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:39,349 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:41,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:32:41,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:32:41,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:32:42,313 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:42,714 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:42,714 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:42,714 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:42,719 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:43,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:43,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:46,527 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:46,528 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:46,528 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:46,532 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:47,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:47,358 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:47,533 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:51,898 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:51,899 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:32:51,899 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:51,904 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:52,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:32:52,904 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:53,363 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:32:56,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:32:56,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:32:56,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:32:58,353 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:00,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:00,364 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:00,364 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:00,369 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:00,369 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:01,373 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:03,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:04,787 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:04,788 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:04,788 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:04,793 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:05,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:05,377 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:06,935 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:33:08,936 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:09,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:09,065 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:09,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:09,070 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:09,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:11,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:33:11,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:33:11,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:33:11,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:14,341 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:15,563 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:15,564 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:15,564 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:15,568 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:16,385 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:17,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:19,606 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:19,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:19,824 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:19,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:19,829 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:20,390 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:21,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:24,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:25,504 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:25,505 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:25,505 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:25,510 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:26,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:33:26,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:33:26,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:33:26,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:27,399 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:30,340 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:31,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:31,770 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:31,770 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:31,775 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:32,401 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:33,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:35,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:36,457 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:36,458 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:36,458 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:36,463 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:36,934 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:33:37,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:37,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:40,431 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:40,431 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:40,432 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:40,438 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:41,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:33:41,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:33:41,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:33:41,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:41,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:41,412 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:44,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:44,303 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:44,303 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:44,307 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:44,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:45,416 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:46,346 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:49,666 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:49,667 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:49,667 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:49,672 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:50,419 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:51,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:33:51,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:55,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:55,294 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:33:55,294 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:55,299 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:55,424 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:33:56,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:33:56,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:33:56,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:33:57,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:57,427 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:01,410 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:01,411 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:01,411 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:01,417 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:01,429 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:02,454 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:03,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:05,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:05,241 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:05,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:05,246 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:05,433 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:06,939 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:34:07,436 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:07,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:09,071 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:09,072 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:09,072 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:09,077 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:09,437 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:11,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:34:11,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:34:11,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:34:11,440 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:12,663 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:12,664 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:12,664 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:12,669 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:13,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:13,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:13,706 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:16,678 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:16,679 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:16,679 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:16,684 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:17,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:17,448 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:19,684 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:21,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:21,597 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:21,598 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:21,602 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:22,452 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:23,454 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:25,601 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:25,601 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:25,602 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:25,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:25,607 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:26,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:34:26,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:34:26,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:34:26,456 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:27,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:30,606 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:30,607 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:30,607 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:30,607 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:30,612 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:31,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:31,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:34,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:34,256 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:34,256 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:34,261 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:34,465 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:35,468 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:36,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:36,939 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:34:37,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:37,999 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:37,999 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:38,004 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:38,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:39,472 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:41,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:34:41,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:34:41,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:34:41,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:41,990 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:41,991 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:41,991 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:41,995 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:42,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:43,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:47,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:48,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:48,259 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:48,259 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:48,264 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:48,479 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:49,481 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:52,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:52,779 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:52,780 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:52,780 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:52,785 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:53,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:53,485 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:56,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:34:56,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:34:56,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:34:56,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:56,769 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:34:56,769 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:56,774 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:57,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:34:57,490 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:34:57,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:00,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:00,571 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:00,571 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:00,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:01,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:01,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:03,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:05,540 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:05,541 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:05,541 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:05,546 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:06,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:06,940 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:35:07,500 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:08,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:10,027 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:10,027 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:10,028 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:10,033 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:10,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:11,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:35:11,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:35:11,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:35:11,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:14,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:14,214 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:14,214 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:14,214 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:14,219 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:14,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:15,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:19,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:20,110 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:20,110 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:20,111 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:20,116 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:20,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:21,515 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:23,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:23,683 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:23,683 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:23,688 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:24,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:24,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:25,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:26,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:35:26,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:35:26,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:35:29,889 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:29,889 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:29,890 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:29,890 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:29,895 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:30,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:31,525 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:33,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:33,897 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:33,897 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:33,902 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:34,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:34,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:35,529 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:36,942 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:35:37,768 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:37,769 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:37,769 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:37,774 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:38,531 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:39,533 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:40,774 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:41,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:35:41,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:35:41,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:35:46,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:46,384 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:46,384 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:46,385 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:46,390 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:46,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:47,540 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:51,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:51,049 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:51,049 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:51,054 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:51,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:51,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:35:52,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:56,059 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:56,060 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:35:56,060 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:56,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:35:56,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:35:56,065 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:56,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:35:56,547 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:35:57,360 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:57,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:01,460 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:01,461 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:01,461 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:01,465 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:01,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:02,502 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:03,556 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:05,424 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:05,424 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:05,425 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:05,430 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:05,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:06,945 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:36:07,560 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:07,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:10,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:10,591 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:10,591 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:10,596 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:11,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:36:11,062 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:36:11,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:36:11,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:11,565 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:13,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:15,732 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:15,733 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:15,733 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:15,738 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:16,568 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:17,571 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:18,774 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:20,135 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:20,135 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:20,135 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:20,140 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:20,574 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:21,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:24,141 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:24,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:24,601 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:24,601 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:24,606 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:25,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:25,581 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:26,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:36:26,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:36:26,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:36:29,288 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:29,327 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:29,328 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:29,328 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:29,332 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:29,583 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:31,586 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:34,370 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:34,539 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:34,542 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:34,542 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:34,547 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:34,588 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:35,591 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:36,947 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:36:39,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:39,187 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:39,187 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:39,192 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:39,595 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:39,595 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:40,192 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:41,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:36:41,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:36:41,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:36:42,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:42,951 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:42,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:42,956 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:43,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:43,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:45,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:47,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:47,814 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:47,814 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:47,819 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:48,602 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:49,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:51,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:52,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:52,450 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:52,451 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:52,455 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:52,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:53,609 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:36:56,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:36:56,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:36:56,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:36:56,883 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:56,883 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:36:56,884 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:56,884 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:56,889 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:57,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:36:57,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:00,629 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:00,630 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:00,630 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:00,635 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:01,619 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:01,619 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:02,636 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:04,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:04,883 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:04,884 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:04,888 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:05,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:05,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:06,949 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:37:07,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:08,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:08,683 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:08,683 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:08,688 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:09,627 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:09,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:11,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:37:11,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:37:11,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:37:12,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:12,036 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:12,036 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:12,041 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:12,629 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:13,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:13,632 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:18,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:20,369 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:20,370 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:20,370 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:20,376 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:20,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:21,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:23,376 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:24,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:24,952 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:24,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:24,957 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:25,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:25,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:26,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:37:26,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:37:26,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:37:29,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:29,978 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:29,979 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:29,979 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:29,989 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:30,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:31,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:33,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:33,962 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:33,962 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:33,967 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:34,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:34,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:35,654 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:36,954 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:37:38,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:38,285 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:38,285 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:38,290 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:38,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:39,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:40,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:41,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:37:41,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:37:41,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:37:46,273 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:47,109 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:47,110 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:47,110 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:47,115 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:47,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:47,665 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:51,397 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:51,398 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:51,398 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:51,399 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:51,404 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:51,668 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:53,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:55,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:55,331 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:55,331 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:55,336 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:55,672 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:37:56,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:37:56,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:37:56,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:37:57,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:57,675 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:37:59,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:59,289 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:37:59,289 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:59,294 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:59,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:01,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:02,334 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:03,784 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:03,785 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:03,785 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:03,790 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:04,683 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:05,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:06,953 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:38:07,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:07,934 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:07,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:07,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:07,939 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:08,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:09,689 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:11,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:38:11,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:38:11,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:38:11,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:11,681 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:11,682 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:11,686 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:11,690 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:13,693 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:13,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:15,445 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:15,446 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:15,446 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:15,451 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:15,694 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:17,697 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:19,416 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:19,417 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:19,418 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:19,418 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:19,422 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:19,700 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:21,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:24,423 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:24,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:24,464 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:24,464 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:24,469 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:24,704 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:25,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:26,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:38:26,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:38:26,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:38:29,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:29,230 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:29,231 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:29,235 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:29,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:29,712 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:30,274 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:33,812 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:33,813 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:33,813 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:33,818 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:34,715 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:35,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:35,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:36,958 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:38:37,983 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:37,983 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:37,984 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:37,988 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:38,719 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:39,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:40,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:41,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:38:41,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:38:41,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:38:42,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:42,972 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:42,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:42,977 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:43,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:43,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:46,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:47,171 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:47,172 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:47,172 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:47,178 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:47,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:47,729 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:51,178 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:54,256 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:54,257 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:54,257 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:54,262 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:54,733 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:55,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:38:56,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:38:56,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:38:56,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:38:56,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:57,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:57,992 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:38:57,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:57,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:58,737 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:38:59,740 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:02,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:02,354 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:02,355 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:02,355 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:02,360 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:02,743 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:03,745 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:06,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:06,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:06,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:06,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:06,747 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:06,957 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:39:07,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:07,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:11,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:39:11,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:39:11,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:39:11,310 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:11,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:11,325 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:11,330 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:11,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:11,753 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:13,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:16,760 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:16,760 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:16,760 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:16,765 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:17,759 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:17,759 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:18,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:21,157 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:21,157 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:21,158 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:21,162 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:21,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:21,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:24,163 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:26,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:39:26,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:39:26,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:39:29,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:30,456 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:30,457 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:30,457 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:30,463 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:30,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:31,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:34,502 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:35,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:35,546 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:35,546 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:35,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:35,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:36,962 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:39:37,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:39,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:40,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:40,826 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:40,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:40,831 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:41,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:39:41,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:39:41,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:39:41,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:41,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:44,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:44,690 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:44,690 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:44,695 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:44,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:45,734 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:45,786 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:48,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:48,478 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:48,478 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:48,483 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:48,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:49,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:51,484 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:53,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:53,292 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:39:53,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:53,297 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:53,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:39:53,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:39:56,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:39:56,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:39:56,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:39:57,350 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:01,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:01,738 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:01,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:01,743 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:01,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:02,782 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:03,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:06,960 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:40:07,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:08,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:08,321 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:08,321 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:08,326 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:08,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:09,808 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:11,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:40:11,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:40:11,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:40:12,155 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:12,155 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:12,155 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:12,160 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:12,810 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:13,198 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:13,812 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:16,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:16,237 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:16,237 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:16,242 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:16,814 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:17,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:18,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:20,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:20,751 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:20,752 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:20,757 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:20,819 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:21,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:23,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:26,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:40:26,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:40:26,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:40:26,390 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:26,391 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:26,391 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:26,396 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:26,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:27,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:29,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:30,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:30,583 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:30,583 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:30,588 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:30,829 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:31,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:34,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:34,190 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:34,191 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:34,196 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:34,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:35,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:35,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:36,962 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:40:40,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:41,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:40:41,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:40:41,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:40:45,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:45,982 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:45,982 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:45,982 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:45,987 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:46,843 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:47,846 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:51,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:51,075 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:51,076 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:51,076 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:51,081 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:51,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:51,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:55,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:55,020 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:55,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:55,025 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:55,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:55,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:40:56,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:40:56,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:40:56,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:40:56,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:59,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:59,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:40:59,278 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:59,283 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:59,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:40:59,859 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:01,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:04,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:04,724 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:04,724 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:04,729 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:04,863 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:05,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:06,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:06,967 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:41:08,474 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:08,475 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:08,475 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:08,480 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:08,867 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:09,869 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:11,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:41:11,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:41:11,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:41:11,963 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:11,964 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:11,964 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:11,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:11,969 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:12,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:13,874 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:17,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:18,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:18,374 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:18,374 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:18,380 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:18,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:19,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:22,380 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:23,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:23,579 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:23,580 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:23,585 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:23,883 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:25,886 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:26,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:41:26,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:41:26,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:41:27,812 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:27,812 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:27,813 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:27,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:27,818 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:27,888 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:29,891 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:32,854 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:33,667 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:33,667 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:33,668 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:33,673 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:33,893 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:35,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:36,966 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:41:37,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:38,786 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:38,787 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:38,787 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:38,792 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:38,898 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:39,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:41,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:41:41,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:41:41,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:41:42,815 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:42,816 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:42,816 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:42,821 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:42,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:43,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:43,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:47,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:47,420 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:47,420 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:47,425 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:47,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:47,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:49,426 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:51,573 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:51,573 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:41:51,574 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:51,579 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:51,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:41:51,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:41:54,579 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:56,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:41:56,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:41:56,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:42:00,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:00,025 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:00,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:00,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:00,030 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:00,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:01,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:04,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:04,034 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:04,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:04,039 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:04,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:05,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:05,927 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:06,971 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:42:08,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:08,272 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:08,273 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:08,277 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:08,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:09,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:10,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:11,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:42:11,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:42:11,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:42:12,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:12,615 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:12,615 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:12,620 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:12,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:13,936 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:15,658 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:16,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:16,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:16,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:16,474 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:16,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:17,940 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:20,412 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:20,413 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:20,413 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:20,418 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:20,943 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:21,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:21,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:24,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:24,649 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:24,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:24,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:24,947 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:25,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:26,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:42:26,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:42:26,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:42:27,321 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:28,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:28,773 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:28,774 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:28,779 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:28,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:29,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:32,818 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:34,679 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:34,680 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:34,680 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:34,685 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:34,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:35,960 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:36,970 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:42:37,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:39,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:39,404 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:39,404 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:39,409 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:39,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:39,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:41,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:42:41,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:42:41,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:42:43,289 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:45,271 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:45,271 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:45,272 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:45,276 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:45,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:45,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:48,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:49,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:49,008 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:49,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:49,014 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:49,974 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:49,974 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:54,015 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:54,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:54,726 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:54,726 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:54,731 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:54,977 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:42:55,979 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:42:56,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:42:56,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:42:56,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:42:59,347 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:59,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:59,827 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:42:59,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:59,832 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:59,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:01,986 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:04,766 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:04,766 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:04,767 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:04,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:04,772 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:04,988 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:05,990 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:06,976 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:43:09,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:09,925 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:09,925 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:09,925 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:09,932 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:09,992 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:11,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:43:11,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:43:11,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:43:11,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:15,270 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:18,364 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:18,365 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:18,365 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:18,370 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:19,000 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:20,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:20,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:25,406 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:26,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:43:26,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:43:26,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:43:27,548 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:27,549 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:27,549 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:27,554 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:28,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:28,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:30,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:33,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:33,084 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:33,084 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:33,089 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:34,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:34,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:36,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:36,975 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:43:37,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:37,882 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:37,882 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:37,887 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:38,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:40,023 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:41,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:43:41,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:43:41,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:43:41,299 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:41,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:41,968 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:41,968 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:41,973 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:42,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:44,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:47,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:47,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:47,039 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:47,039 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:47,044 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:48,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:48,031 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:51,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:51,239 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:51,239 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:51,244 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:52,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:52,036 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:52,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:55,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:55,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:55,789 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:55,794 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:56,038 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:43:56,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:43:56,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:43:56,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:43:57,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:58,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:43:59,818 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:59,818 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:43:59,818 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:59,824 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:00,042 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:02,047 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:02,862 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:06,847 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:06,848 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:06,848 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:06,853 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:06,979 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:44:07,050 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:07,981 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:08,053 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:10,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:10,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:10,809 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:10,814 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:11,055 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:11,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:44:11,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:44:11,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:44:12,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:13,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:14,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:14,832 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:14,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:14,837 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:15,059 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:16,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:18,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:19,131 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:19,132 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:19,132 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:19,137 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:20,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:20,067 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:24,138 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:26,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:44:26,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:44:26,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:44:27,411 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:27,412 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:27,412 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:27,417 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:28,075 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:28,075 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:29,454 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:31,968 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:31,969 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:31,969 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:31,974 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:32,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:34,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:34,975 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:36,979 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:44:39,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:40,848 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:40,849 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:40,849 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:40,855 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:41,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:44:41,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:44:41,067 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:44:41,084 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:42,087 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:44,764 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:44,765 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:44,765 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:44,769 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:45,088 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:45,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:46,091 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:49,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:49,620 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:49,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:49,625 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:50,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:50,095 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:51,626 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:53,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:53,993 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:53,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:53,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:54,097 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:56,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:44:56,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:44:56,067 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:44:56,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:44:57,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:57,675 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:57,676 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:44:57,676 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:57,681 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:58,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:44:58,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:01,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:01,882 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:01,882 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:01,887 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:02,108 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:02,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:04,111 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:06,012 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:06,013 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:06,013 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:06,018 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:06,112 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:06,981 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:45:07,982 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:08,115 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:10,421 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:10,422 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:10,422 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:10,427 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:11,117 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:11,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:45:11,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:45:11,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:45:12,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:13,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:15,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:15,415 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:15,415 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:15,421 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:16,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:16,124 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:18,458 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:20,702 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:20,703 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:20,704 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:20,708 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:21,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:22,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:23,709 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:24,415 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:24,415 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:24,416 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:24,421 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:25,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:26,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:26,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:45:26,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:45:26,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:45:29,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:32,975 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:32,975 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:32,976 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:32,981 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:33,140 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:34,143 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:35,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:36,983 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:45:37,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:37,830 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:37,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:37,835 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:38,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:38,147 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:40,836 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:41,659 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:41,660 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:41,660 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:41,665 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:41,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:45:41,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:45:41,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:45:42,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:42,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:45,648 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:45,648 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:45,649 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:45,654 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:46,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:46,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:46,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:49,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:49,575 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:49,575 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:49,580 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:50,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:50,160 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:52,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:53,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:53,315 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:45:53,315 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:53,320 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:54,164 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:45:54,164 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:45:56,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:45:56,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:45:56,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:45:58,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:01,385 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:01,386 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:01,386 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:01,391 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:02,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:02,172 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:03,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:05,147 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:05,148 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:05,148 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:05,154 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:05,173 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:06,176 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:06,985 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:46:08,987 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:11,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:46:11,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:46:11,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:46:12,315 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:12,316 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:12,316 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:12,321 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:13,180 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:14,183 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:14,358 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:17,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:17,195 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:17,195 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:17,200 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:18,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:18,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:20,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:21,146 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:21,147 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:21,147 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:21,152 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:21,190 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:22,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:24,788 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:24,789 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:24,789 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:24,794 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:25,194 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:25,795 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:26,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:26,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:46:26,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:46:26,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:46:28,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:28,758 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:28,758 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:28,763 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:29,199 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:30,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:30,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:33,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:33,070 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:33,070 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:33,075 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:33,204 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:34,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:36,076 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:36,987 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:46:37,272 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:37,273 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:37,273 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:37,279 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:38,210 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:38,211 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:41,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:41,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:46:41,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:46:41,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:46:42,798 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:42,798 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:42,799 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:42,804 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:43,214 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:44,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:46,726 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:46,727 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:46,728 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:46,728 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:46,733 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:47,218 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:48,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:50,718 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:50,719 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:50,719 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:50,724 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:51,222 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:52,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:52,724 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:54,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:54,865 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:54,866 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:54,871 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:55,226 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:46:56,228 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:46:56,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:46:56,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:46:56,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:46:58,107 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:58,548 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:58,548 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:46:58,549 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:58,554 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:59,230 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:00,233 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:47:03,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:03,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:03,925 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:47:03,925 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:03,930 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:04,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:04,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:47:06,989 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:47:08,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:11,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:47:11,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:47:11,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:47:12,456 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:12,457 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:47:12,457 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:12,462 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:13,244 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:14,246 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:47:14,498 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:17,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:17,464 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:47:17,465 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:17,470 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:18,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:18,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:47:20,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:22,817 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:22,818 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:47:22,818 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:22,823 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:23,255 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:24,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:47:25,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:26,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:47:26,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:47:26,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:47:27,198 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:27,199 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:47:27,199 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:27,204 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:27,259 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:28,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:47:31,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:32,728 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:32,729 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:47:32,729 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:32,734 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:33,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:34,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:47:36,735 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:36,994 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:47:38,884 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:38,885 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:47:38,885 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:38,890 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:39,270 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:40,272 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:47:41,824 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:47:41,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:41,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:47:41,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:47:46,970 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:46,971 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:47:46,971 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:46,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:46,976 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:47,276 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:48,278 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:47:52,014 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:52,396 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:52,397 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:47:52,397 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:52,402 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:53,281 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:54,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:47:56,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:56,624 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:47:56,624 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:56,629 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:56,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:47:56,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:47:56,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:47:57,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:57,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:47:58,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:02,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:02,874 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:02,875 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:02,875 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:02,880 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:03,293 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:04,295 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:06,993 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:48:07,757 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:07,758 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:07,758 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:07,758 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:07,763 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:08,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:08,299 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:11,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:48:11,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:48:11,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:48:11,966 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:12,085 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:12,085 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:12,090 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:12,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:13,091 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:14,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:16,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:16,509 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:16,510 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:16,515 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:17,306 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:18,309 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:18,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:23,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:23,920 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:23,920 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:23,921 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:23,925 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:24,315 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:24,316 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:26,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:48:26,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:48:26,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:48:28,208 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:28,209 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:28,209 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:28,214 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:28,319 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:29,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:30,321 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:32,180 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:32,181 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:32,181 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:32,186 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:32,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:34,326 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:35,187 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:36,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:36,340 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:36,340 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:36,345 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:36,998 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:48:37,327 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:38,330 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:39,907 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:39,908 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:39,909 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:39,914 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:40,333 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:40,333 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:40,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:41,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:48:41,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:48:41,825 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:48:46,057 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:48,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:48,441 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:48,442 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:48,447 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:49,339 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:50,341 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:51,486 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:52,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:52,259 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:52,259 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:52,264 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:52,342 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:54,345 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:48:56,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:56,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:48:56,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:48:56,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:48:57,704 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:57,705 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:48:57,705 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:57,710 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:58,349 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:48:58,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:02,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:02,094 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:02,094 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:02,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:02,099 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:02,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:04,356 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:06,998 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:49:07,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:07,361 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:07,362 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:07,362 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:07,367 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:08,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:08,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:11,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:49:11,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:49:11,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:49:12,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:12,033 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:12,033 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:12,039 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:12,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:12,365 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:13,039 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:17,807 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:17,808 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:17,809 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:17,813 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:18,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:18,370 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:18,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:22,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:22,224 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:22,224 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:22,229 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:22,373 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:24,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:24,376 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:26,150 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:26,151 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:26,151 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:26,156 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:26,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:26,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:49:26,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:49:26,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:49:28,381 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:30,090 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:34,718 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:34,719 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:34,719 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:34,724 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:35,385 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:35,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:36,388 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:37,003 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:49:39,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:39,517 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:39,517 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:39,522 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:40,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:40,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:41,523 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:41,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:49:41,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:49:41,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:49:43,906 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:43,906 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:43,906 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:43,911 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:44,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:44,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:46,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:51,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:53,175 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:53,176 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:53,176 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:53,181 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:53,402 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:54,404 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:49:56,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:49:56,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:49:56,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:49:57,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:57,174 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:57,175 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:49:57,175 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:57,180 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:57,406 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:49:58,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:00,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:00,935 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:00,935 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:00,940 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:01,410 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:02,414 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:02,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:05,856 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:05,857 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:05,857 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:05,862 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:06,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:06,418 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:07,002 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:50:08,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:11,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:50:11,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:50:11,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:50:12,143 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:12,144 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:12,144 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:12,150 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:12,422 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:13,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:14,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:16,813 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:16,814 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:16,815 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:16,819 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:17,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:18,429 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:18,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:20,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:20,618 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:20,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:20,623 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:21,432 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:22,434 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:24,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:24,333 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:24,333 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:24,333 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:24,338 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:24,436 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:26,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:26,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:50:26,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:50:26,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:50:28,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:28,061 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:28,061 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:28,066 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:28,442 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:28,443 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:30,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:31,975 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:31,976 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:31,977 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:31,981 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:32,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:32,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:35,982 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:37,004 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:50:38,185 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:38,186 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:38,186 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:38,191 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:38,450 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:40,453 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:41,192 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:41,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:50:41,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:50:41,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:50:42,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:42,218 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:42,219 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:42,224 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:42,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:44,457 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:46,262 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:47,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:47,453 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:47,454 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:47,459 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:47,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:48,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:51,460 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:56,072 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:56,073 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:56,073 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:56,078 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:56,469 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:50:56,469 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:50:56,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:50:56,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:50:56,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:56,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:50:59,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:59,823 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:50:59,823 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:59,828 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:00,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:00,473 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:01,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:03,239 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:03,240 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:03,240 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:03,245 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:03,476 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:04,478 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:07,006 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:51:07,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:09,549 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:09,549 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:09,550 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:09,555 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:10,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:10,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:11,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:51:11,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:51:11,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:51:12,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:13,765 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:13,765 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:13,766 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:13,771 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:14,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:14,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:17,810 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:20,932 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:20,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:20,933 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:20,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:21,493 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:22,496 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:22,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:24,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:24,867 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:24,867 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:24,872 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:25,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:26,500 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:26,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:51:26,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:51:26,826 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:51:28,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:28,666 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:28,667 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:28,667 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:28,672 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:29,503 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:30,505 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:32,472 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:32,473 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:32,473 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:32,478 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:32,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:33,479 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:34,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:37,008 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:51:39,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:39,610 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:39,611 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:39,611 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:39,616 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:40,514 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:40,515 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:41,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:51:41,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:51:41,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:51:43,845 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:43,845 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:43,846 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:43,850 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:44,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:44,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:44,890 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:48,240 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:48,240 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:48,241 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:48,246 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:48,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:48,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:50,247 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:54,155 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:54,155 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:54,156 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:54,161 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:54,528 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:51:54,528 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:51:56,161 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:56,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:51:56,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:51:56,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:51:59,033 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:59,034 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:51:59,034 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:59,039 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:59,531 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:00,534 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:02,081 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:03,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:03,451 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:03,451 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:03,456 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:03,537 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:04,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:07,010 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:52:07,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:07,623 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:07,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:07,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:07,628 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:08,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:08,544 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:11,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:52:11,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:52:11,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:52:12,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:12,308 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:12,308 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:12,313 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:12,546 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:13,350 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:14,549 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:17,841 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:17,842 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:17,842 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:17,847 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:18,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:18,553 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:18,848 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:22,476 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:22,477 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:22,477 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:22,482 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:22,556 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:24,482 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:24,559 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:26,207 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:26,208 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:26,208 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:26,213 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:26,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:26,563 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:26,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:52:26,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:52:26,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:52:29,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:29,617 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:29,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:29,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:29,622 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:30,568 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:30,568 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:34,662 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:35,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:35,617 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:35,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:35,622 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:36,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:36,574 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:37,012 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:52:39,796 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:39,797 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:39,797 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:39,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:39,802 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:40,578 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:40,578 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:41,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:52:41,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:52:41,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:52:43,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:43,903 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:43,903 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:43,908 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:44,582 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:44,582 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:44,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:49,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:49,510 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:49,510 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:49,515 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:49,585 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:50,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:50,588 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:53,123 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:53,124 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:53,125 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:53,129 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:53,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:54,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:52:56,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:56,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:52:56,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:52:56,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:52:57,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:57,702 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:52:57,702 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:57,707 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:58,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:52:58,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:01,745 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:02,988 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:02,989 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:02,989 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:02,994 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:03,600 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:04,604 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:06,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:06,623 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:06,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:06,628 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:07,015 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:53:07,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:07,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:08,608 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:11,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:11,308 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:11,308 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:11,313 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:11,610 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:11,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:53:11,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:53:11,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:53:12,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:12,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:17,052 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:17,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:17,323 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:17,323 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:17,328 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:17,615 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:18,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:21,435 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:21,436 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:21,436 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:21,441 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:21,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:22,442 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:22,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:25,843 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:25,844 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:25,844 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:25,849 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:26,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:26,628 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:26,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:53:26,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:53:26,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:53:28,086 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:29,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:29,828 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:29,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:29,833 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:30,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:30,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:33,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:33,367 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:33,367 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:33,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:33,372 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:33,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:34,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:37,016 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:53:37,769 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:37,770 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:37,770 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:37,775 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:38,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:38,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:38,776 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:41,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:53:41,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:53:41,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:53:42,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:42,081 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:42,081 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:42,086 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:42,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:42,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:44,087 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:45,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:45,454 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:45,454 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:45,459 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:45,647 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:46,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:49,459 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:49,627 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:49,628 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:49,628 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:49,633 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:49,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:50,654 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:53,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:53,555 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:53,555 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:53,560 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:53,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:53:54,561 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:54,658 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:53:56,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:53:56,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:53:56,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:53:59,705 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:59,705 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:53:59,706 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:59,706 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:59,711 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:00,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:00,664 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:03,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:03,997 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:03,997 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:04,002 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:04,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:04,669 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:05,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:07,022 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:54:08,172 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:08,173 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:08,173 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:08,178 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:08,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:08,673 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:10,179 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:11,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:54:11,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:54:11,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:54:14,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:14,210 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:14,210 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:14,215 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:14,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:14,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:15,254 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:18,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:18,748 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:18,749 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:18,753 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:19,681 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:20,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:20,754 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:23,661 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:23,662 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:23,662 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:23,667 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:23,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:24,689 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:26,668 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:26,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:54:26,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:54:26,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:54:28,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:28,043 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:28,044 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:28,049 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:28,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:28,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:32,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:32,038 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:32,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:32,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:32,043 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:32,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:32,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:36,319 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:36,320 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:36,320 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:36,325 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:36,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:36,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:37,021 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:54:38,022 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:41,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:54:41,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:54:41,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:54:42,867 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:42,867 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:42,867 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:42,872 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:43,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:43,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:44,710 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:47,879 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:47,879 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:47,880 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:47,885 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:48,714 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:48,714 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:49,885 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:54,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:56,234 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:56,234 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:54:56,235 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:56,241 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:56,720 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:54:56,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:54:56,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:54:56,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:54:56,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:55:00,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:01,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:01,234 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:01,234 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:01,239 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:01,723 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:02,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:05,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:05,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:05,317 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:05,318 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:05,323 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:05,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:06,732 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:07,025 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:55:08,987 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:08,988 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:08,988 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:08,993 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:09,734 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:10,736 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:10,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:11,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:55:11,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:55:11,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:55:16,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:16,656 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:16,657 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:16,657 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:16,662 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:16,740 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:18,744 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:21,294 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:21,294 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:21,295 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:21,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:21,300 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:21,748 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:22,750 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:25,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:25,303 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:25,303 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:25,308 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:25,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:26,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:26,754 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:26,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:55:26,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:55:26,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:55:29,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:29,085 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:29,085 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:29,094 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:29,757 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:30,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:32,130 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:33,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:33,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:33,523 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:33,528 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:33,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:34,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:37,025 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:55:37,489 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:37,490 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:37,490 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:37,490 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:37,495 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:37,766 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:38,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:41,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:55:41,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:55:41,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:55:41,915 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:42,117 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:42,117 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:42,122 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:42,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:42,773 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:43,122 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:47,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:47,011 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:47,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:47,016 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:47,775 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:48,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:49,016 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:50,728 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:50,728 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:50,729 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:50,733 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:50,779 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:52,782 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:54,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:54,038 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:54,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:54,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:54,043 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:54,785 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:55:54,785 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:55:56,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:55:56,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:55:56,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:55:59,125 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:59,736 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:59,736 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:55:59,736 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:59,741 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:59,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:00,790 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:03,737 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:03,737 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:03,738 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:03,742 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:03,793 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:04,743 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:04,796 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:07,029 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:56:08,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:08,990 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:08,990 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:08,995 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:09,799 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:09,996 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:10,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:11,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:56:11,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:56:11,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:56:14,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:14,106 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:14,106 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:14,111 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:14,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:14,805 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:15,150 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:19,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:19,003 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:19,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:19,008 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:19,808 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:20,812 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:21,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:23,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:23,043 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:23,043 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:23,049 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:23,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:24,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:26,049 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:26,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:56:26,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:56:26,828 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:56:27,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:27,197 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:27,198 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:27,202 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:27,818 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:28,821 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:31,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:32,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:32,150 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:32,150 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:32,155 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:32,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:32,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:37,029 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:56:37,030 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:38,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:38,281 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:38,281 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:38,287 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:38,830 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:38,830 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:41,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:56:41,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:56:41,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:56:42,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:44,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:44,267 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:44,267 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:44,271 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:44,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:44,836 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:47,310 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:48,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:48,096 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:48,097 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:48,102 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:48,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:48,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:53,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:54,536 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:54,537 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:56:54,537 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:54,543 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:54,845 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:56:54,845 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:56:56,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:56:56,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:56:56,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:56:59,078 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:01,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:01,910 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:01,910 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:01,915 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:02,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:02,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:04,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:06,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:06,074 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:06,074 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:06,078 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:06,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:06,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:07,034 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:57:10,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:10,323 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:10,324 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:10,325 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:10,329 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:10,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:10,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:11,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:57:11,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:57:11,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:57:13,792 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:13,793 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:13,793 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:13,798 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:13,863 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:14,865 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:15,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:18,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:18,154 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:18,154 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:18,159 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:18,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:18,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:21,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:24,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:24,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:24,259 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:24,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:24,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:24,876 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:26,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:26,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:57:26,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:57:26,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:57:28,433 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:28,434 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:28,434 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:28,439 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:28,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:28,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:31,478 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:32,673 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:32,674 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:32,674 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:32,679 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:32,884 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:34,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:36,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:37,033 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:57:37,341 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:37,342 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:37,342 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:37,347 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:37,889 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:38,891 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:41,828 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:57:41,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:41,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:57:41,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:57:42,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:42,374 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:42,374 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:42,379 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:42,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:42,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:47,418 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:47,595 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:47,596 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:47,596 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:47,601 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:47,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:48,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:51,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:51,716 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:51,716 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:51,721 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:51,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:52,721 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:52,906 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:56,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:56,107 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:57:56,107 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:56,112 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:56,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:57:56,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:57:56,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:57:56,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:57:56,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:57:58,099 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:00,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:00,331 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:00,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:00,337 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:00,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:00,914 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:03,374 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:04,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:04,231 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:04,231 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:04,236 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:04,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:04,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:07,035 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:58:09,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:10,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:10,025 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:10,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:10,030 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:10,924 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:10,925 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:11,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:58:11,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:58:11,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:58:14,040 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:14,207 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:14,207 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:14,208 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:14,213 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:14,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:14,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:18,388 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:18,389 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:18,389 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:18,394 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:18,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:18,933 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:19,394 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:22,999 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:23,000 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:23,000 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:23,005 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:23,937 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:24,940 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:25,006 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:26,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:58:26,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:58:26,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:58:27,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:27,224 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:27,224 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:27,229 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:27,942 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:28,945 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:30,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:32,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:32,022 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:32,022 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:32,027 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:32,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:32,950 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:35,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:35,591 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:35,591 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:35,591 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:35,596 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:35,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:36,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:37,037 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:58:40,501 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:40,502 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:40,502 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:40,507 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:40,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:40,958 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:41,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:41,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:58:41,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:58:41,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:58:44,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:44,279 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:44,279 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:44,284 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:44,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:44,962 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:47,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:48,950 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:48,951 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:48,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:48,956 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:48,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:50,967 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:52,941 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:52,941 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:52,942 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:52,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:52,947 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:52,968 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:54,971 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:58:56,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:58:56,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:58:56,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:58:58,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:58,443 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:58,443 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:58:58,443 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:58,448 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:58,976 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:58:58,976 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:02,307 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:02,308 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:02,308 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:02,313 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:02,981 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:59:02,982 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:03,314 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:07,040 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:59:07,388 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:07,389 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:07,389 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:07,394 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:07,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:59:08,394 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:08,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:11,250 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:11,251 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:11,251 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:11,256 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:11,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:59:11,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:59:11,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:59:11,988 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:59:12,991 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:14,021 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:17,387 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:17,388 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:17,388 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:17,393 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:17,994 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:59:18,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:19,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:22,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:22,203 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:22,203 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:22,208 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:23,002 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:59:23,002 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:25,209 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:26,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:59:26,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:59:26,829 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:59:30,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:30,575 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:30,575 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:30,575 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:30,580 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:31,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:59:31,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:35,180 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:35,181 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:35,181 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:35,186 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:36,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:59:36,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:37,014 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:37,041 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 17:59:40,513 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:40,514 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:40,514 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:40,519 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:41,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:59:41,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:41,519 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:41,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:59:41,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:59:41,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:59:45,571 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:45,572 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:45,572 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:45,577 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:46,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:59:46,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:47,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:50,608 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:50,609 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:50,609 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:50,615 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:51,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 17:59:51,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 17:59:51,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:56,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:56,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:59:56,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:59:56,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:59:59,717 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:59,718 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 17:59:59,718 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:59,725 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:00,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:01,035 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:01,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:03,551 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:03,552 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:03,552 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:03,558 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:04,038 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:05,041 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:07,043 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:00:07,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:07,736 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:07,737 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:07,737 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:07,742 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:08,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:09,048 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:11,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:00:11,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:00:11,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:00:12,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:12,464 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:12,465 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:12,465 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:12,470 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:13,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:13,053 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:16,135 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:16,136 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:16,137 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:16,141 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:17,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:17,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:17,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:22,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:23,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:23,634 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:23,634 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:23,640 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:24,062 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:25,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:26,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:00:26,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:00:26,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:00:28,098 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:28,170 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:28,170 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:28,170 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:28,175 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:29,069 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:29,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:32,784 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:32,784 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:32,785 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:32,790 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:33,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:33,074 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:33,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:37,048 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:00:39,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:41,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:41,330 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:41,331 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:41,335 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:41,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:00:41,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:00:41,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:00:42,078 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:43,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:44,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:45,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:45,462 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:45,462 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:45,467 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:46,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:47,085 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:49,096 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:49,096 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:49,097 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:49,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:49,102 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:50,087 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:51,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:52,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:52,924 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:00:52,925 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:52,930 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:53,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:00:54,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:55,093 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:00:56,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:00:56,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:00:56,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:01:00,060 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:00,242 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:00,242 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:00,242 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:00,247 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:01,099 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:01,099 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:04,664 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:04,665 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:04,665 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:04,670 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:05,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:05,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:05,670 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:07,048 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:01:08,687 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:08,688 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:08,688 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:08,693 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:09,108 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:09,108 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:10,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:11,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:01:11,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:01:11,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:01:14,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:14,541 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:14,542 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:14,547 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:15,113 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:15,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:16,586 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:18,662 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:18,663 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:18,663 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:18,668 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:19,118 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:19,118 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:21,669 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:22,843 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:22,844 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:22,844 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:22,850 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:23,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:23,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:26,148 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:26,148 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:26,149 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:26,154 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:26,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:01:26,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:26,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:01:26,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:01:27,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:27,127 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:30,005 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:30,005 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:30,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:30,010 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:30,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:31,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:32,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:35,371 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:35,372 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:35,372 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:35,377 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:36,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:37,052 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:01:37,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:37,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:39,525 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:39,526 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:39,526 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:39,531 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:40,139 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:41,142 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:41,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:01:41,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:01:41,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:01:42,126 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:44,144 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:44,145 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:44,145 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:44,150 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:45,146 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:45,146 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:47,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:48,786 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:48,787 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:48,787 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:48,792 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:49,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:49,151 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:52,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:53,219 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:53,220 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:53,220 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:53,225 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:54,154 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:55,156 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:01:56,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:01:56,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:01:56,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:01:57,121 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:57,121 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:01:57,122 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:57,127 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:57,157 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:01:58,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:59,160 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:01,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:01,563 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:01,564 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:01,568 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:02,162 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:03,166 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:03,569 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:05,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:05,286 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:05,287 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:05,291 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:06,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:07,052 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:02:07,170 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:09,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:11,747 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:11,748 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:11,749 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:11,754 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:11,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:02:11,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:02:11,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:02:12,173 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:13,175 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:14,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:15,884 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:15,884 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:15,885 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:15,890 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:16,177 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:17,179 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:19,926 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:24,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:24,482 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:24,482 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:24,488 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:25,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:25,188 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:25,488 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:26,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:02:26,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:02:26,830 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:02:28,858 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:28,858 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:28,858 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:28,863 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:29,192 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:29,193 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:30,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:32,431 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:32,431 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:32,432 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:32,436 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:33,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:33,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:36,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:37,057 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:02:38,236 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:38,237 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:38,237 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:38,242 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:39,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:39,202 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:41,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:02:41,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:41,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:02:41,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:02:42,956 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:42,956 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:42,957 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:42,961 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:43,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:43,206 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:46,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:47,842 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:47,842 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:47,843 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:47,848 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:48,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:49,211 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:51,928 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:51,928 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:51,929 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:51,934 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:52,213 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:52,934 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:53,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:56,329 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:56,329 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:02:56,329 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:56,334 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:56,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:02:56,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:02:56,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:02:57,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:02:57,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:02:58,118 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:00,310 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:00,311 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:00,311 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:00,316 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:01,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:01,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:03,354 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:05,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:05,063 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:05,064 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:05,068 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:05,228 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:07,056 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:03:07,231 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:09,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:09,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:09,267 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:09,267 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:09,272 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:10,232 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:11,235 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:11,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:03:11,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:03:11,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:03:14,120 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:17,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:17,233 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:17,234 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:17,238 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:17,239 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:19,242 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:19,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:21,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:21,409 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:21,409 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:21,414 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:22,245 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:23,247 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:24,415 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:25,854 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:25,854 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:25,855 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:25,860 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:26,249 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:26,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:03:26,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:03:26,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:03:27,252 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:29,871 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:29,872 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:29,872 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:29,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:29,877 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:30,254 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:31,257 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:34,815 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:34,815 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:34,816 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:34,821 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:35,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:35,261 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:35,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:37,061 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:03:40,085 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:40,086 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:40,086 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:40,092 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:40,264 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:41,092 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:41,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:41,830 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:03:41,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:03:41,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:03:43,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:43,839 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:43,839 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:43,844 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:44,268 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:45,271 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:46,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:51,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:52,372 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:52,373 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:52,373 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:52,378 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:53,277 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:03:53,277 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:03:56,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:03:56,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:03:56,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:03:57,046 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:59,675 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:59,676 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:03:59,676 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:59,681 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:00,282 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:01,284 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:02,718 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:04,452 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:04,453 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:04,453 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:04,458 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:05,290 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:05,290 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:07,061 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:04:08,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:08,387 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:08,388 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:08,388 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:08,393 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:09,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:09,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:11,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:04:11,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:04:11,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:04:12,148 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:12,148 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:12,148 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:12,153 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:12,296 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:13,190 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:13,298 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:15,903 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:15,904 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:15,904 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:15,910 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:16,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:17,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:18,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:19,680 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:19,681 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:19,681 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:19,686 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:20,304 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:21,308 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:24,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:24,187 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:24,187 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:24,187 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:24,193 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:24,310 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:25,312 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:26,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:04:26,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:04:26,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:04:28,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:28,571 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:28,571 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:28,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:29,318 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:29,318 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:29,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:33,359 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:33,360 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:33,360 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:33,365 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:34,321 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:35,323 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:35,365 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:37,065 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:04:39,261 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:39,262 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:39,262 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:39,267 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:39,325 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:41,267 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:41,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:41,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:04:41,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:04:41,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:04:45,470 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:45,471 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:45,471 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:45,476 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:46,332 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:46,514 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:47,334 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:49,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:49,509 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:49,509 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:49,514 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:50,336 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:51,338 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:04:51,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:56,034 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:56,035 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:04:56,035 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:56,040 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:56,341 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:04:56,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:04:56,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:04:56,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:56,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:04:57,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:00,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:00,650 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:00,650 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:00,655 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:01,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:01,347 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:02,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:07,065 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:05:07,922 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:07,923 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:07,923 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:07,923 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:07,928 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:08,353 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:09,356 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:11,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:05:11,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:05:11,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:05:13,036 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:16,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:16,187 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:16,187 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:16,192 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:16,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:17,362 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:18,230 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:19,798 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:19,799 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:19,799 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:19,804 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:20,364 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:21,368 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:23,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:26,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:05:26,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:05:26,831 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:05:27,347 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:27,347 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:27,348 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:27,353 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:27,371 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:29,375 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:29,390 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:32,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:32,439 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:32,439 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:32,444 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:33,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:33,379 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:34,444 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:36,930 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:36,931 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:36,931 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:36,936 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:37,067 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:05:37,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:37,383 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:40,068 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:41,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:05:41,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:05:41,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:05:44,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:44,138 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:44,138 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:44,143 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:44,387 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:45,181 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:45,390 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:49,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:49,642 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:49,642 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:49,647 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:50,392 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:50,647 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:51,395 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:54,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:54,025 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:54,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:54,030 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:54,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:55,399 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:05:56,031 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:56,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:05:56,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:05:56,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:05:58,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:58,683 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:05:58,683 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:58,688 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:59,403 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:05:59,403 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:01,725 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:02,457 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:02,458 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:02,458 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:02,464 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:03,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:03,409 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:06,232 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:06,233 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:06,233 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:06,238 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:06,410 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:07,069 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:06:07,071 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:07,413 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:11,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:11,056 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:11,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:11,061 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:11,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:11,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:11,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:06:11,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:06:11,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:06:12,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:14,419 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:14,420 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:14,420 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:14,425 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:15,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:15,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:17,461 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:17,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:17,951 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:17,951 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:17,957 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:18,423 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:19,425 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:22,957 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:23,170 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:23,171 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:23,171 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:23,176 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:23,431 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:23,431 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:26,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:06:26,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:06:26,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:06:26,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:27,103 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:27,104 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:27,109 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:27,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:27,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:28,110 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:31,592 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:31,593 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:31,593 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:31,598 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:32,439 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:33,441 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:33,598 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:37,071 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:06:37,424 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:37,424 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:37,425 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:37,430 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:37,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:39,430 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:39,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:41,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:06:41,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:06:41,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:06:41,876 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:42,120 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:42,120 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:42,125 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:42,449 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:43,451 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:45,126 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:45,677 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:45,678 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:45,678 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:45,683 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:46,453 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:47,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:50,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:54,219 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:54,220 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:54,220 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:54,225 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:54,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:55,459 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:06:56,225 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:56,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:06:56,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:06:56,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:06:58,276 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:58,277 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:06:58,277 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:58,282 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:58,461 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:06:59,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:01,283 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:04,559 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:04,559 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:04,560 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:04,565 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:05,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:05,470 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:06,565 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:07,076 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:07:08,895 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:08,896 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:08,896 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:08,901 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:09,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:09,474 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:11,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:11,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:07:11,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:07:11,832 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:07:14,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:14,118 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:14,118 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:14,123 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:14,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:15,479 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:17,124 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:17,951 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:17,952 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:17,952 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:17,957 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:18,481 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:19,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:22,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:24,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:24,012 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:24,012 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:24,017 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:24,488 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:25,490 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:26,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:07:26,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:07:26,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:07:28,084 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:29,366 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:29,367 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:29,367 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:29,372 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:29,493 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:31,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:33,373 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:34,775 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:34,776 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:34,776 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:34,782 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:35,501 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:35,501 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:37,075 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:07:38,080 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:38,081 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:38,081 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:38,087 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:38,503 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:39,088 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:39,505 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:41,794 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:41,795 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:41,795 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:41,800 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:41,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:07:41,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:07:41,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:07:42,507 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:43,509 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:45,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:46,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:46,279 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:46,280 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:46,284 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:46,511 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:47,513 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:50,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:50,051 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:50,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:50,056 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:50,515 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:51,056 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:51,517 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:53,733 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:53,734 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:53,734 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:53,739 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:54,519 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:55,522 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:07:56,740 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:56,832 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:07:56,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:07:56,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:07:58,055 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:58,056 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:07:58,056 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:58,061 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:58,523 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:07:59,526 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:02,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:03,045 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:03,046 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:03,046 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:03,051 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:03,531 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:03,531 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:06,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:06,887 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:06,888 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:06,892 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:07,080 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:08:07,081 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:07,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:07,536 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:11,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:11,729 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:11,730 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:11,734 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:11,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:08:11,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:08:11,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:08:12,538 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:13,063 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:13,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:16,383 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:16,384 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:16,384 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:16,389 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:16,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:17,545 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:18,390 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:21,450 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:21,451 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:21,451 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:21,456 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:21,549 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:23,457 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:23,552 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:26,609 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:26,610 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:26,610 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:26,616 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:26,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:08:26,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:08:26,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:08:27,556 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:27,556 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:29,043 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:31,786 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:31,787 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:31,787 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:31,792 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:32,560 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:33,562 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:34,793 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:36,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:36,225 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:36,225 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:36,230 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:36,564 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:37,079 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:08:37,566 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:40,081 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:41,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:08:41,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:08:41,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:08:42,681 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:42,682 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:42,682 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:42,688 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:43,572 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:43,572 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:45,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:46,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:46,548 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:46,548 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:46,553 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:46,573 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:47,576 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:50,580 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:50,581 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:50,581 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:50,586 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:51,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:51,580 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:51,586 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:54,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:54,820 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:08:54,820 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:54,825 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:55,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:08:55,584 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:08:56,825 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:56,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:08:56,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:08:56,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:09:00,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:00,321 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:00,321 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:00,326 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:00,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:01,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:02,327 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:04,219 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:04,220 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:04,220 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:04,225 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:04,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:05,595 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:07,084 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:09:08,086 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:08,086 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:08,086 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:08,087 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:08,092 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:08,596 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:09,599 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:11,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:09:11,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:09:11,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:09:12,711 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:12,712 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:12,712 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:12,717 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:13,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:13,603 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:13,718 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:17,544 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:17,544 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:17,545 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:17,549 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:17,605 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:19,550 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:19,608 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:21,218 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:21,219 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:21,219 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:21,224 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:21,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:21,613 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:25,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:25,048 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:25,048 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:25,048 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:25,053 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:25,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:25,617 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:26,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:09:26,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:09:26,833 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:09:28,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:28,716 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:28,716 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:28,721 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:29,622 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:29,622 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:30,722 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:32,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:32,896 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:32,897 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:32,901 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:33,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:33,626 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:35,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:36,780 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:36,781 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:36,781 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:36,786 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:37,083 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:09:37,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:37,630 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:41,085 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:41,714 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:41,715 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:41,715 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:41,720 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:41,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:09:41,834 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:09:41,876 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:09:42,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:43,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:47,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:47,069 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:47,070 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:47,070 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:47,075 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:47,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:47,640 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:51,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:51,260 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:51,260 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:51,265 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:51,643 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:51,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:52,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:56,062 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:56,063 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:09:56,063 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:56,068 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:56,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:09:56,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:09:56,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:09:56,834 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:09:57,649 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:09:58,123 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:02,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:02,806 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:02,806 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:02,811 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:03,655 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:03,655 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:03,811 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:07,088 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:10:07,369 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:07,370 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:07,370 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:07,375 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:07,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:07,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:09,375 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:11,079 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:11,080 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:11,080 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:11,085 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:11,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:11,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:11,833 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:10:11,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:10:11,834 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:10:14,800 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:14,801 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:14,801 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:14,801 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:14,806 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:15,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:15,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:19,522 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:19,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:19,523 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:19,527 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:19,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:19,671 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:20,528 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:25,528 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:25,696 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:25,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:25,697 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:25,702 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:26,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:26,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:10:26,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:10:26,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:10:27,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:31,095 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:34,019 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:34,020 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:34,020 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:34,025 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:34,684 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:35,686 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:37,061 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:37,088 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:10:37,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:37,973 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:37,973 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:37,979 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:38,688 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:39,691 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:41,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:10:41,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:10:41,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:10:42,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:42,157 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:42,157 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:42,158 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:42,162 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:42,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:43,695 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:47,163 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:50,723 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:50,724 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:50,724 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:50,729 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:51,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:51,701 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:52,729 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:54,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:54,601 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:54,601 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:54,606 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:54,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:55,705 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:10:56,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:10:56,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:10:56,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:10:58,121 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:58,536 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:58,537 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:10:58,537 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:58,542 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:58,707 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:10:59,709 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:02,284 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:02,285 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:02,285 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:02,290 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:02,711 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:03,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:03,715 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:07,093 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:11:08,350 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:08,351 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:08,351 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:08,351 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:08,356 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:08,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:09,720 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:11,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:11:11,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:11:11,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:11:12,339 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:12,339 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:12,340 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:12,345 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:12,722 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:13,724 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:14,345 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:17,041 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:17,042 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:17,042 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:17,047 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:17,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:17,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:20,047 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:21,645 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:21,646 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:21,647 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:21,651 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:21,732 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:23,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:25,652 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:25,742 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:25,743 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:25,743 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:25,749 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:26,737 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:26,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:11:26,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:11:26,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:11:27,740 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:29,491 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:29,492 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:29,493 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:29,497 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:29,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:31,498 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:31,744 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:33,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:33,302 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:33,303 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:33,307 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:33,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:35,749 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:37,092 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:11:37,093 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:40,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:40,464 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:40,464 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:40,469 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:40,752 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:41,754 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:41,834 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:11:41,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:11:41,835 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:11:42,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:44,408 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:44,409 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:44,409 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:44,414 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:44,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:45,758 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:47,414 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:48,115 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:48,116 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:48,116 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:48,121 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:48,760 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:49,762 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:52,357 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:52,358 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:52,358 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:52,363 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:52,764 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:53,364 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:53,766 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:56,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:56,195 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:11:56,195 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:56,201 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:56,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:11:57,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:11:57,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:11:57,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:11:57,770 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:11:58,913 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:02,969 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:02,970 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:02,970 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:02,975 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:03,777 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:03,777 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:04,013 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:07,094 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:12:07,137 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:07,138 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:07,138 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:07,143 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:07,781 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:07,781 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:09,144 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:11,381 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:11,382 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:11,382 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:11,388 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:11,783 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:12,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:12:12,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:12:12,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:12:13,786 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:14,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:16,730 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:16,730 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:16,731 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:16,735 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:16,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:17,791 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:20,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:20,259 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:20,260 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:20,260 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:20,264 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:20,792 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:21,796 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:25,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:25,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:25,467 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:25,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:25,472 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:25,798 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:27,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:12:27,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:12:27,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:12:27,802 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:30,901 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:33,903 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:33,904 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:33,904 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:33,909 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:34,806 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:35,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:35,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:37,096 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:12:37,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:37,918 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:37,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:37,923 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:38,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:39,813 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:41,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:42,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:42,117 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:42,118 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:42,123 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:42,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:12:42,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:12:42,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:12:42,815 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:43,818 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:47,335 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:47,336 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:47,336 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:47,336 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:47,341 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:47,820 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:49,823 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:52,377 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:52,400 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:52,400 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:52,401 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:52,405 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:52,824 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:53,826 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:12:56,486 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:56,486 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:12:56,487 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:56,492 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:56,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:12:57,493 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:57,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:12:57,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:12:57,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:12:57,830 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:00,541 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:00,542 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:00,542 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:00,547 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:00,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:01,835 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:02,585 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:05,362 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:05,363 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:05,363 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:05,368 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:05,838 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:07,098 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:13:07,842 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:08,100 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:09,734 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:09,735 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:09,735 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:09,740 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:09,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:11,849 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:12,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:13:12,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:13:12,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:13:13,747 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:13,748 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:13,748 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:13,748 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:13,756 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:13,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:15,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:17,536 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:17,539 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:17,540 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:17,544 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:17,856 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:19,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:19,860 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:23,734 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:23,735 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:23,735 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:23,740 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:23,864 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:24,741 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:25,869 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:27,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:13:27,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:13:27,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:13:27,876 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:27,876 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:27,876 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:27,881 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:28,871 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:29,875 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:29,917 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:32,633 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:32,633 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:32,634 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:32,639 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:32,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:33,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:35,639 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:36,952 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:36,953 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:36,953 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:36,958 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:37,100 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:13:37,886 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:37,886 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:41,102 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:42,615 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:13:42,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:13:42,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:13:46,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:47,983 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:47,984 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:47,984 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:47,989 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:48,899 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:49,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:52,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:52,389 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:52,390 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:52,390 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:52,395 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:52,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:53,909 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:55,755 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:55,756 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:55,756 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:55,761 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:55,911 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:13:57,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:13:57,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:57,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:13:57,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:13:57,916 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:13:59,763 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:59,763 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:13:59,764 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:59,769 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:59,917 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:01,922 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:02,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:05,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:05,615 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:05,615 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:05,620 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:05,926 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:07,102 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:14:07,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:08,104 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:12,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:14:12,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:14:12,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:14:13,916 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:14,136 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:14,137 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:14,137 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:14,142 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:14,938 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:15,942 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:18,111 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:18,112 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:18,112 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:18,117 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:18,944 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:19,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:19,948 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:22,609 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:22,610 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:22,610 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:22,615 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:22,951 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:23,954 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:24,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:26,449 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:26,450 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:26,450 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:26,455 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:26,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:27,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:14:27,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:14:27,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:14:27,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:29,871 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:30,826 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:30,827 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:30,827 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:30,832 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:30,961 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:31,965 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:34,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:34,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:34,946 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:34,947 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:34,951 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:34,966 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:35,970 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:37,104 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:14:38,877 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:38,878 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:38,878 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:38,883 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:38,972 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:39,884 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:39,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:42,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:14:42,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:14:42,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:14:43,009 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:43,010 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:43,010 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:43,014 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:43,981 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:43,981 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:45,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:47,018 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:47,019 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:47,019 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:47,024 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:47,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:47,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:51,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:51,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:51,225 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:51,225 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:51,230 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:51,993 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:51,993 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:14:56,231 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:56,902 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:56,903 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:14:56,903 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:56,908 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:56,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:14:57,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:14:57,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:14:57,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:14:57,998 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:01,523 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:01,524 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:01,524 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:01,524 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:01,529 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:02,000 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:04,005 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:06,565 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:06,977 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:06,978 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:06,978 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:06,983 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:07,007 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:07,107 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:15:08,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:12,109 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:12,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:15:12,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:15:12,616 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:15:12,768 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:12,913 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:12,913 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:12,918 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:13,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:14,014 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:17,359 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:17,360 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:17,361 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:17,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:17,366 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:18,018 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:18,019 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:22,367 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:25,662 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:25,663 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:25,663 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:25,668 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:26,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:27,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:15:27,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:27,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:15:27,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:15:28,027 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:29,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:29,888 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:29,888 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:29,893 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:30,029 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:32,032 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:32,929 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:33,898 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:33,898 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:33,899 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:33,903 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:34,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:36,036 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:37,112 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:15:38,113 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:41,446 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:41,447 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:41,447 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:41,452 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:42,039 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:42,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:15:42,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:15:42,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:15:43,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:44,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:45,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:45,295 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:45,296 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:45,300 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:46,045 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:46,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:49,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:49,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:49,961 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:49,961 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:49,966 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:50,048 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:52,050 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:15:54,967 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:56,572 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:56,573 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:15:56,573 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:56,578 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:57,054 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:15:57,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:15:57,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:15:57,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:15:58,056 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:00,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:01,026 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:01,027 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:01,027 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:01,032 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:01,058 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:02,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:04,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:04,702 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:04,703 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:04,707 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:05,063 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:06,066 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:06,708 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:07,111 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:16:12,113 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:12,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:16:12,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:16:12,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:16:13,053 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:13,054 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:13,054 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:13,059 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:13,070 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:14,072 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:18,097 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:18,139 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:18,140 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:18,140 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:18,145 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:19,075 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:20,077 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:21,508 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:21,509 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:21,509 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:21,514 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:22,080 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:23,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:24,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:25,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:25,323 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:25,323 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:25,328 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:26,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:26,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:27,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:16:27,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:16:27,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:16:28,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:29,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:29,654 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:29,654 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:29,659 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:30,089 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:32,092 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:34,264 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:34,264 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:34,265 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:34,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:34,270 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:35,094 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:36,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:37,116 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:16:39,121 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:39,122 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:39,122 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:39,127 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:40,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:40,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:40,127 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:42,616 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:16:42,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:16:42,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:16:43,233 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:43,234 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:43,234 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:43,239 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:44,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:44,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:45,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:47,003 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:47,004 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:47,004 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:47,009 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:47,106 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:48,108 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:51,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:52,937 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:52,938 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:52,938 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:52,943 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:53,111 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:16:54,114 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:16:56,944 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:57,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:16:57,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:16:57,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:16:59,072 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:59,073 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:16:59,073 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:59,078 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:59,116 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:00,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:02,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:04,587 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:04,588 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:04,588 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:04,593 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:05,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:06,125 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:07,115 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:17:07,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:09,762 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:09,763 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:09,763 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:09,768 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:10,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:12,130 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:12,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:17:12,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:17:12,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:12,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:17:14,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:14,004 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:14,005 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:14,022 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:14,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:16,134 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:17,556 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:17,557 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:17,557 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:17,562 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:18,135 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:18,562 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:20,139 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:21,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:21,548 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:21,548 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:21,553 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:22,141 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:24,144 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:24,553 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:25,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:25,575 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:25,575 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:25,580 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:26,145 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:27,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:17:27,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:17:27,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:17:28,148 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:29,868 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:34,108 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:34,109 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:34,109 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:34,114 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:34,152 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:35,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:36,155 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:37,120 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:17:37,865 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:37,866 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:37,866 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:37,871 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:38,156 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:40,159 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:40,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:42,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:17:42,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:17:42,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:17:43,396 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:43,397 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:43,397 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:43,402 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:44,163 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:44,164 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:46,442 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:47,222 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:47,222 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:47,223 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:47,228 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:48,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:48,168 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:52,228 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:52,312 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:52,313 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:52,313 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:52,318 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:53,171 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:54,173 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:56,245 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:56,246 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:56,246 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:56,251 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:57,175 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:17:57,251 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:57,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:17:57,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:17:57,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:17:58,177 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:17:59,738 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:59,739 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:17:59,739 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:59,744 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:00,178 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:02,181 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:02,782 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:04,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:04,791 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:04,791 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:04,796 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:05,184 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:06,187 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:07,119 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:18:08,121 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:09,448 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:09,449 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:09,450 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:09,454 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:10,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:10,191 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:12,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:18:12,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:18:12,617 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:18:13,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:13,258 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:13,258 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:13,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:13,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:14,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:14,195 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:17,066 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:17,067 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:17,068 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:17,072 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:17,197 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:18,199 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:19,073 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:21,101 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:21,102 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:21,102 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:21,107 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:21,201 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:22,205 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:24,107 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:25,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:25,574 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:25,575 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:25,579 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:26,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:26,209 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:27,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:18:27,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:18:27,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:18:29,854 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:30,065 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:30,065 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:30,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:30,070 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:30,212 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:32,215 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:33,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:33,894 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:33,894 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:33,899 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:34,216 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:34,899 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:36,219 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:37,124 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:18:37,650 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:37,651 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:37,651 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:37,656 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:38,220 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:40,224 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:40,657 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:42,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:18:42,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:18:42,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:18:44,774 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:44,774 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:44,775 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:44,779 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:45,226 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:45,818 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:46,229 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:49,796 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:49,797 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:49,797 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:49,801 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:50,231 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:51,802 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:52,234 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:54,817 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:54,817 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:54,818 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:54,822 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:55,236 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:18:56,238 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:18:56,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:57,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:18:57,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:18:57,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:18:58,558 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:58,558 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:18:58,559 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:58,563 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:59,240 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:00,242 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:02,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:07,037 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:07,038 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:07,038 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:07,043 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:07,123 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:19:07,247 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:08,125 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:08,250 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:11,025 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:11,026 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:11,026 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:11,031 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:11,251 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:12,254 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:12,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:19:12,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:19:12,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:19:13,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:14,355 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:14,356 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:14,356 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:14,361 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:15,255 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:16,258 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:19,115 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:19,116 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:19,116 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:19,116 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:19,121 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:19,259 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:20,262 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:23,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:23,482 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:23,482 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:23,487 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:24,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:24,267 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:24,488 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:27,617 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:19:27,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:19:27,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:19:28,707 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:28,707 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:28,707 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:28,712 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:29,270 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:29,750 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:30,273 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:32,477 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:32,478 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:32,478 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:32,483 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:33,275 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:34,277 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:35,483 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:36,599 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:36,600 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:36,600 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:36,605 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:37,128 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:19:37,279 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:38,281 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:40,751 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:40,752 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:40,752 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:40,752 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:40,757 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:41,283 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:42,286 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:42,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:19:42,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:19:42,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:19:44,590 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:44,591 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:44,591 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:44,596 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:45,288 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:46,290 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:46,634 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:49,257 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:49,257 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:49,258 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:49,263 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:49,292 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:50,294 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:52,263 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:53,325 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:53,325 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:53,326 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:53,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:54,298 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:54,298 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:19:57,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:57,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:19:57,278 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:57,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:57,283 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:57,300 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:19:57,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:19:57,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:19:57,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:19:58,302 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:01,863 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:01,864 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:01,864 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:01,869 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:02,305 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:02,906 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:04,309 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:05,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:05,546 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:05,546 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:05,551 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:06,312 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:06,312 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:07,128 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:20:08,129 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:10,089 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:10,089 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:10,090 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:10,094 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:10,314 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:12,317 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:12,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:20:12,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:20:12,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:20:13,604 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:13,605 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:13,605 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:13,605 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:13,610 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:14,320 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:14,321 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:17,538 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:17,539 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:17,539 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:17,544 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:18,324 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:18,325 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:19,545 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:23,004 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:23,005 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:23,006 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:23,010 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:23,329 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:24,331 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:25,011 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:27,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:27,362 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:27,362 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:27,367 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:27,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:20:27,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:20:27,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:20:28,335 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:28,335 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:30,855 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:32,547 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:32,548 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:32,548 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:32,552 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:33,339 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:34,341 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:36,401 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:36,402 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:36,403 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:36,403 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:36,407 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:37,133 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:20:37,343 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:38,345 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:42,134 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:42,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:42,578 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:42,578 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:42,583 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:42,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:20:42,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:20:42,618 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:20:43,348 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:44,350 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:46,903 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:46,904 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:46,904 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:46,909 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:47,352 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:47,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:48,354 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:52,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:52,280 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:20:52,280 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:52,285 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:52,357 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:20:53,286 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:54,360 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:20:57,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:20:57,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:20:57,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:20:58,855 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:01,395 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:01,395 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:01,396 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:01,400 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:02,366 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:02,367 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:04,438 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:07,132 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:21:09,689 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:09,690 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:09,690 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:09,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:09,695 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:10,374 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:10,374 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:12,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:21:12,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:21:12,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:21:13,655 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:13,655 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:13,656 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:13,660 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:14,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:14,378 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:14,698 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:17,460 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:17,460 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:17,461 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:17,465 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:18,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:18,382 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:20,466 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:21,822 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:21,823 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:21,823 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:21,828 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:22,386 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:24,389 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:25,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:27,259 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:27,260 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:27,260 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:27,264 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:27,390 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:27,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:21:27,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:21:27,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:21:28,393 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:30,914 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:33,179 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:33,179 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:33,179 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:33,184 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:33,396 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:34,399 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:36,222 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:37,134 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:21:41,627 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:41,628 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:41,628 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:41,628 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:41,633 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:42,405 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:42,405 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:42,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:21:42,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:21:42,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:21:46,715 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:46,715 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:46,715 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:46,716 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:46,720 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:47,408 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:48,410 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:51,428 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:51,429 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:51,429 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:51,434 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:52,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:52,415 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:52,435 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:55,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:55,897 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:55,897 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:55,902 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:56,417 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:21:57,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:57,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:21:57,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:21:57,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:21:58,420 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:21:59,837 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:59,838 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:21:59,838 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:59,843 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:00,421 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:02,424 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:02,882 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:04,223 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:04,223 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:04,224 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:04,228 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:04,426 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:06,429 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:07,136 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:22:08,138 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:09,152 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:09,152 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:09,153 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:09,157 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:09,431 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:10,433 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:12,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:22:12,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:22:12,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:22:13,007 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:13,008 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:13,008 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:13,013 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:13,435 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:14,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:14,438 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:16,945 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:16,946 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:16,946 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:16,952 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:17,440 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:18,442 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:19,953 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:20,632 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:20,633 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:20,633 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:20,638 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:21,444 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:22,447 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:25,290 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:25,291 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:25,291 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:25,292 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:25,296 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:25,449 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:26,452 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:27,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:22:27,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:22:27,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:22:29,994 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:29,995 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:29,995 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:30,000 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:30,455 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:31,038 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:32,458 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:34,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:34,697 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:34,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:34,702 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:35,460 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:36,462 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:36,703 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:37,138 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:22:39,155 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:39,156 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:39,156 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:39,161 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:39,464 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:40,466 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:42,162 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:42,618 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:22:42,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:22:42,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:22:43,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:43,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:43,278 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:43,283 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:43,468 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:44,471 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:47,322 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:47,422 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:47,423 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:47,423 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:47,428 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:47,472 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:48,475 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:50,686 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:50,687 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:50,687 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:50,692 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:51,477 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:52,480 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:52,692 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:55,058 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:55,059 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:55,059 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:55,064 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:55,481 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:22:56,484 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:22:57,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:22:57,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:22:57,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:22:57,851 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:59,915 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:59,918 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:22:59,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:59,923 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:00,486 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:02,489 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:02,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:05,849 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:05,850 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:05,850 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:05,855 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:06,494 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:06,495 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:07,140 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:23:08,142 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:10,265 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:10,266 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:10,266 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:10,271 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:10,497 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:12,500 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:12,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:23:12,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:23:12,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:23:13,831 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:14,916 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:14,916 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:14,917 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:14,922 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:15,502 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:16,504 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:18,958 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:19,188 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:19,189 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:19,189 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:19,194 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:19,506 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:20,508 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:24,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:27,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:23:27,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:23:27,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:23:27,838 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:27,921 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:27,921 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:27,926 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:28,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:28,516 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:29,926 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:31,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:31,791 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:31,792 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:31,797 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:32,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:32,521 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:35,797 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:37,142 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:23:38,303 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:38,304 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:38,304 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:38,309 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:38,524 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:40,527 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:41,309 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:41,734 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:41,735 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:41,735 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:41,740 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:42,530 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:42,531 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:42,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:23:42,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:23:42,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:23:45,892 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:45,893 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:45,893 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:45,898 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:46,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:46,535 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:46,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:49,634 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:49,635 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:49,635 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:49,640 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:50,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:50,539 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:52,641 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:53,496 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:53,496 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:23:53,497 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:53,501 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:53,541 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:23:54,543 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:23:57,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:23:57,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:23:57,619 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:23:57,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:01,896 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:01,897 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:01,897 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:01,902 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:02,549 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:02,550 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:02,942 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:06,921 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:06,922 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:06,922 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:06,927 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:07,147 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:24:07,554 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:08,149 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:08,557 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:11,624 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:11,625 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:11,625 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:11,630 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:12,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:12,561 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:12,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:24:12,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:24:12,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:24:13,885 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:16,010 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:16,011 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:16,011 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:16,016 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:16,563 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:18,566 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:19,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:20,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:20,993 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:20,993 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:20,998 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:21,568 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:22,572 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:24,998 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:27,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:24:27,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:24:27,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:24:27,843 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:27,854 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:27,854 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:27,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:28,577 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:28,578 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:30,860 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:31,767 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:31,767 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:31,767 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:31,772 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:32,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:32,587 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:36,079 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:36,080 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:36,080 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:36,080 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:36,085 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:36,589 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:37,147 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:24:38,592 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:40,154 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:40,155 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:40,155 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:40,160 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:40,593 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:41,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:42,597 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:42,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:24:42,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:24:42,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:24:45,497 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:45,498 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:45,498 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:45,503 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:45,598 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:46,542 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:46,601 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:51,542 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:53,692 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:53,693 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:53,693 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:53,698 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:54,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:54,607 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:24:56,699 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:57,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:24:57,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:24:57,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:24:57,827 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:57,861 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:24:57,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:57,866 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:58,611 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:24:58,612 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:01,867 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:03,127 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:03,127 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:03,127 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:03,132 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:03,614 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:04,618 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:07,133 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:07,151 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:25:08,088 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:08,089 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:08,089 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:08,094 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:08,620 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:10,623 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:12,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:12,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:25:12,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:25:12,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:25:15,893 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:15,894 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:15,894 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:15,899 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:16,627 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:16,627 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:17,938 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:20,331 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:20,331 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:20,332 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:20,336 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:20,629 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:22,633 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:23,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:25,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:25,161 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:25,161 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:25,167 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:25,635 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:26,637 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:27,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:25:27,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:25:27,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:25:28,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:28,975 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:28,976 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:28,976 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:28,980 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:29,639 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:30,642 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:32,980 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:32,981 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:32,981 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:32,986 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:33,644 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:33,986 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:34,646 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:37,002 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:37,003 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:37,003 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:37,008 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:37,151 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:25:37,648 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:38,650 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:39,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:41,050 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:41,051 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:41,051 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:41,056 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:41,652 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:42,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:25:42,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:25:42,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:25:42,654 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:44,603 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:44,604 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:44,604 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:44,604 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:44,609 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:44,656 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:46,659 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:48,853 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:48,854 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:48,854 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:48,859 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:49,660 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:49,859 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:50,663 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:53,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:53,824 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:53,824 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:53,829 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:54,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:25:54,667 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:25:55,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:57,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:25:57,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:25:57,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:25:59,296 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:59,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:25:59,297 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:59,302 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:59,670 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:26:00,672 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:26:01,338 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:04,559 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:04,560 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:26:04,560 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:04,565 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:04,676 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:26:06,565 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:06,679 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:26:07,156 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:26:08,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:08,338 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:26:08,338 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:08,343 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:08,680 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:26:10,683 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:26:12,344 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:12,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:26:12,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:26:12,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:26:13,126 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:13,126 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:26:13,127 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:13,131 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:13,685 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:26:14,687 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:26:18,170 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:18,326 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:18,327 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:26:18,327 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:18,331 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:18,689 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:26:20,692 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:26:23,332 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:25,361 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:25,361 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:26:25,362 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:25,366 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:25,696 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:26:26,699 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:26:27,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:26:27,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:26:27,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:26:28,867 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:33,697 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:33,698 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:26:33,698 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:33,703 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:33,703 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:26:34,706 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:26:34,742 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:37,155 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:26:40,157 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:42,237 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:42,237 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:26:42,238 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:42,242 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:42,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:26:42,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:26:42,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:26:42,710 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:26:44,713 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:26:45,852 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:46,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:46,235 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:26:46,235 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:46,240 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:46,715 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:26:48,717 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:26:49,509 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:49,510 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:26:49,510 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:49,515 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:49,718 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:26:50,721 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:26:51,515 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:56,516 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:57,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:26:57,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:26:57,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:26:58,521 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:58,522 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:26:58,522 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:58,527 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:58,725 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:00,728 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:01,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:02,866 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:02,867 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:02,867 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:02,871 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:03,730 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:04,733 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:06,872 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:07,157 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:27:07,277 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:07,278 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:07,278 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:07,283 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:07,735 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:08,737 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:11,808 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:11,812 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:11,812 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:11,817 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:12,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:27:12,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:27:12,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:12,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:27:12,741 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:12,742 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:17,843 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:20,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:20,196 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:20,197 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:20,202 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:20,746 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:22,750 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:23,238 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:24,585 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:24,586 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:24,586 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:24,591 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:24,751 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:26,754 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:27,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:27:27,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:27:27,621 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:27:28,224 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:28,225 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:28,225 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:28,230 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:28,266 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:28,756 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:30,759 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:32,943 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:32,944 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:32,944 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:32,949 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:33,761 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:33,949 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:34,763 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:37,159 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:27:37,437 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:37,437 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:37,438 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:37,442 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:37,765 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:38,768 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:39,443 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:41,569 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:41,569 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:41,570 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:41,575 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:41,769 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:42,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:27:42,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:27:42,621 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:27:42,772 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:44,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:49,829 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:49,867 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:49,867 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:49,867 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:49,872 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:50,778 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:50,779 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:54,910 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:55,662 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:55,663 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:27:55,663 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:55,668 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:55,781 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:27:56,784 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:27:57,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:27:57,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:27:57,621 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:28:00,159 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:00,159 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:00,159 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:00,160 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:00,164 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:00,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:00,788 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:05,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:07,161 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:28:08,683 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:08,684 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:08,684 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:08,689 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:08,794 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:10,690 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:10,797 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:12,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:28:12,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:28:12,621 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:28:13,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:13,620 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:13,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:13,625 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:13,798 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:14,801 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:16,662 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:18,886 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:18,887 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:18,887 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:18,892 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:19,803 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:20,804 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:21,892 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:23,293 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:23,294 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:23,294 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:23,299 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:23,807 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:24,809 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:27,285 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:27,286 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:27,286 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:27,287 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:27,291 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:27,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:28:27,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:28:27,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:28:27,811 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:28,814 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:31,235 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:31,236 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:31,236 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:31,240 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:31,816 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:32,818 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:33,241 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:35,463 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:35,464 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:35,464 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:35,469 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:35,820 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:36,822 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:37,164 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:28:39,167 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:40,688 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:40,689 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:40,689 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:40,694 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:40,825 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:42,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:28:42,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:28:42,621 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:28:42,828 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:44,864 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:44,964 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:44,965 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:44,965 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:44,970 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:45,829 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:46,832 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:49,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:50,481 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:50,482 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:50,482 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:50,487 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:50,834 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:28:52,837 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:28:55,488 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:57,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:28:57,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:28:57,621 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:28:58,798 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:58,799 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:28:58,799 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:58,804 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:58,840 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:00,805 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:00,843 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:02,646 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:02,647 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:02,647 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:02,652 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:02,844 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:04,848 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:06,653 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:06,790 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:06,791 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:06,791 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:06,796 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:06,850 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:07,166 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:29:08,853 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:11,166 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:11,167 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:11,167 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:11,172 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:11,854 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:12,172 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:12,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:29:12,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:29:12,621 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:29:12,857 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:15,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:15,296 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:15,296 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:15,301 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:15,858 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:16,861 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:17,301 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:22,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:23,120 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:23,121 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:23,121 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:23,126 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:23,866 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:24,869 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:27,569 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:27,570 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:27,570 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:27,571 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:27,575 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:27,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:29:27,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:29:27,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:29:27,870 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:28,873 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:31,959 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:31,959 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:31,960 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:31,964 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:32,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:32,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:32,965 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:36,017 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:36,018 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:36,018 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:36,023 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:36,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:36,882 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:37,168 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:29:38,170 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:41,278 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:41,279 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:41,279 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:41,284 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:41,885 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:42,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:29:42,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:29:42,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:29:42,887 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:43,835 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:46,164 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:46,165 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:46,165 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:46,170 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:46,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:46,892 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:49,170 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:50,330 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:50,330 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:50,330 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:50,336 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:50,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:50,896 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:54,337 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:54,692 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:54,693 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:54,693 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:54,698 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:54,898 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:29:56,901 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:29:57,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:29:57,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:29:57,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:29:59,295 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:59,295 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:29:59,296 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:59,301 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:59,903 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:00,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:00,905 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:05,302 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:07,170 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:30:07,574 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:07,575 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:07,575 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:07,580 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:07,910 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:08,913 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:10,581 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:12,621 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:30:12,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:30:12,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:30:15,908 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:15,979 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:15,979 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:15,979 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:15,984 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:16,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:16,919 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:20,153 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:20,153 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:20,154 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:20,158 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:20,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:20,923 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:21,159 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:25,933 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:25,933 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:25,934 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:25,938 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:26,929 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:26,930 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:26,939 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:27,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:30:27,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:30:27,664 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:30:29,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:29,806 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:29,806 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:29,811 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:29,931 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:30,934 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:32,850 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:34,024 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:34,025 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:34,025 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:34,030 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:34,938 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:34,938 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:37,175 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:30:38,176 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:38,526 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:38,527 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:38,527 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:38,532 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:38,940 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:40,943 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:42,371 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:42,371 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:42,372 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:42,377 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:42,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:30:42,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:30:42,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:30:42,946 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:42,947 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:43,279 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:48,280 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:48,313 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:48,314 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:48,314 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:48,319 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:48,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:48,952 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:53,320 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:53,518 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:53,519 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:53,519 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:53,524 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:53,955 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:54,957 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:30:57,487 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:57,488 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:30:57,488 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:57,493 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:57,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:30:57,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:30:57,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:30:57,959 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:30:58,847 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:58,961 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:02,764 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:02,764 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:02,765 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:02,770 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:02,964 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:04,806 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:04,968 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:07,175 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:31:07,983 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:07,983 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:07,984 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:07,988 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:08,972 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:08,972 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:09,989 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:12,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:31:12,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:31:12,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:31:13,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:13,614 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:13,615 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:13,619 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:13,975 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:14,976 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:15,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:17,816 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:17,817 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:17,817 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:17,822 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:17,978 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:18,980 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:20,823 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:22,588 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:22,588 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:22,589 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:22,593 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:22,984 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:24,987 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:26,564 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:26,565 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:26,565 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:26,565 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:26,570 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:26,988 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:27,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:31:27,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:31:27,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:31:28,991 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:31,619 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:31,620 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:31,620 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:31,620 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:31,625 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:31,993 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:32,996 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:35,368 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:35,369 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:35,369 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:35,374 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:35,997 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:37,000 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:37,179 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:31:37,181 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:40,291 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:40,292 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:40,292 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:40,297 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:41,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:41,004 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:42,298 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:42,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:31:42,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:31:42,622 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:31:44,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:44,178 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:44,178 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:44,183 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:45,008 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:45,009 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:47,971 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:47,971 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:47,972 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:47,972 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:47,976 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:48,010 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:49,012 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:52,749 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:52,750 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:31:52,750 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:52,755 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:53,015 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:31:53,756 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:55,017 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:31:57,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:31:57,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:31:57,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:31:58,875 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:00,909 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:00,910 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:00,910 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:00,919 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:01,021 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:03,024 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:03,954 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:04,467 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:04,468 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:04,468 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:04,472 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:05,028 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:05,029 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:07,179 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:32:07,918 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:07,918 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:07,918 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:07,923 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:08,030 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:09,033 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:09,924 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:12,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:32:12,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:32:12,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:32:13,177 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:13,178 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:13,178 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:13,183 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:14,036 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:15,038 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:15,184 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:18,186 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:18,187 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:18,187 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:18,191 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:19,042 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:19,043 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:20,192 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:23,365 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:23,366 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:23,366 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:23,371 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:24,046 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:25,049 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:25,371 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:27,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:32:27,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:32:27,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:32:29,064 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:29,065 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:29,065 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:29,070 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:30,052 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:31,055 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:31,110 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:33,472 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:33,473 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:33,473 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:33,478 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:34,057 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:35,059 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:36,479 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:37,184 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:32:38,694 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:38,695 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:38,695 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:38,700 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:39,061 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:41,064 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:41,701 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:42,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:32:42,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:32:42,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:32:44,510 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:44,511 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:44,511 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:44,516 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:45,068 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:45,069 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:47,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:52,554 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:52,919 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:52,920 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:52,920 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:52,925 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:53,073 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:32:55,076 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:32:57,622 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:57,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:32:57,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:32:57,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:32:59,213 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:59,214 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:32:59,214 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:59,219 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:00,079 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:01,081 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:03,258 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:03,317 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:03,318 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:03,318 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:03,323 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:04,083 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:05,086 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:07,183 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:33:08,600 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:08,601 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:08,601 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:08,602 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:08,606 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:09,090 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:09,091 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:12,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:33:12,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:33:12,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:33:13,054 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:13,055 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:13,055 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:13,060 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:13,093 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:14,098 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:15,096 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:18,527 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:18,528 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:18,528 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:18,533 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:19,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:19,100 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:19,533 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:22,728 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:22,729 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:22,729 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:22,734 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:23,104 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:24,735 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:25,107 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:27,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:33:27,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:33:27,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:33:28,639 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:28,639 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:28,639 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:28,644 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:29,109 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:30,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:31,113 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:35,682 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:37,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:37,115 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:37,115 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:37,120 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:37,186 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:33:38,117 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:39,119 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:41,070 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:41,071 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:41,071 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:41,071 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:41,076 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:41,120 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:42,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:33:42,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:33:42,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:33:43,123 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:45,076 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:45,076 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:45,076 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:45,081 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:45,125 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:46,120 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:47,128 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:48,434 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:48,435 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:48,435 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:48,440 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:49,131 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:49,132 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:51,441 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:52,881 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:52,882 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:33:52,882 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:52,886 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:53,134 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:33:55,137 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:33:56,887 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:57,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:33:57,623 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:33:57,623 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:34:02,841 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:07,188 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:34:08,189 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:13,077 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:34:13,077 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:34:13,316 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:13,578 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:34:18,773 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:23,774 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:28,453 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:34:28,597 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:34:28,597 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:34:28,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:33,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:37,191 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:34:39,193 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:44,193 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:49,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:54,196 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:59,197 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:02,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:35:02,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:35:02,820 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:35:05,083 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:07,192 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:35:10,194 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:15,195 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:17,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:35:17,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:35:17,820 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:35:21,115 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:26,116 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:31,116 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:32,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:35:32,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:35:32,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:35:37,114 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:37,199 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:35:42,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:47,202 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:47,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:35:47,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:35:47,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:35:53,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:58,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:02,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:36:02,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:36:02,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:36:07,821 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:36:07,946 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:36:07,946 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:12,948 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:17,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:36:17,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:36:17,820 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:36:18,106 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:23,107 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:28,107 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:32,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:36:32,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:36:32,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:36:34,074 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:37,199 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:36:39,200 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:44,201 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:47,819 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:36:47,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:36:47,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:36:50,122 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:55,123 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:00,124 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:02,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:37:02,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:37:02,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:37:06,117 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:07,205 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:37:11,207 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:16,207 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:17,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:37:17,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:37:17,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:37:22,150 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:27,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:32,151 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:32,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:37:32,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:37:32,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:37:37,203 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:37:37,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:42,205 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:47,206 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:47,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:37:47,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:37:47,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:37:53,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:58,094 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:02,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:38:02,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:38:02,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:38:03,119 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:07,205 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:38:08,207 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:13,207 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:17,820 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 18:38:17,861 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:38:17,861 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:38:19,163 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:22,985 DEBUG SenderThread:877043 [sender.py:send():382] send: telemetry +2024-03-13 18:38:22,985 DEBUG SenderThread:877043 [sender.py:send():382] send: exit +2024-03-13 18:38:22,986 INFO SenderThread:877043 [sender.py:send_exit():589] handling exit code: 0 +2024-03-13 18:38:22,986 INFO SenderThread:877043 [sender.py:send_exit():591] handling runtime: 25817 +2024-03-13 18:38:22,991 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:22,991 INFO SenderThread:877043 [sender.py:send_exit():597] send defer +2024-03-13 18:38:22,991 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:22,992 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-13 18:38:22,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:22,992 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-13 18:38:22,992 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 1 +2024-03-13 18:38:22,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:22,992 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-13 18:38:22,992 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:22,992 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-13 18:38:22,992 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 2 +2024-03-13 18:38:22,992 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:22,992 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-13 18:38:22,992 INFO HandlerThread:877043 [system_monitor.py:finish():203] Stopping system monitor +2024-03-13 18:38:22,992 DEBUG SystemMonitor:877043 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-13 18:38:22,993 DEBUG SystemMonitor:877043 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-13 18:38:22,994 INFO HandlerThread:877043 [interfaces.py:finish():202] Joined cpu monitor +2024-03-13 18:38:22,996 INFO HandlerThread:877043 [interfaces.py:finish():202] Joined disk monitor +2024-03-13 18:38:23,874 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:38:24,568 INFO HandlerThread:877043 [interfaces.py:finish():202] Joined gpu monitor +2024-03-13 18:38:24,568 INFO HandlerThread:877043 [interfaces.py:finish():202] Joined memory monitor +2024-03-13 18:38:24,568 INFO HandlerThread:877043 [interfaces.py:finish():202] Joined network monitor +2024-03-13 18:38:24,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 18:38:24,568 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:24,569 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-13 18:38:24,569 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 3 +2024-03-13 18:38:24,569 DEBUG SenderThread:877043 [sender.py:send():382] send: stats +2024-03-13 18:38:24,570 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 18:38:24,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:24,570 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-13 18:38:24,570 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:24,571 DEBUG SenderThread:877043 [sender.py:send():382] send: history +2024-03-13 18:38:24,571 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:24,576 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:24,576 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:24,576 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-13 18:38:24,576 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 4 +2024-03-13 18:38:24,576 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:24,576 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-13 18:38:24,576 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:24,577 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-13 18:38:24,577 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 5 +2024-03-13 18:38:24,577 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:24,577 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-13 18:38:24,577 DEBUG SenderThread:877043 [sender.py:send():382] send: summary +2024-03-13 18:38:24,582 INFO SenderThread:877043 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:24,582 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:24,582 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-13 18:38:24,582 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 6 +2024-03-13 18:38:24,582 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:24,583 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-13 18:38:24,583 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:24,583 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-13 18:38:24,585 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:24,814 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 7 +2024-03-13 18:38:24,814 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:24,814 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-13 18:38:24,814 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:24,814 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-13 18:38:24,877 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:38:24,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:38:24,878 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/config.yaml +2024-03-13 18:38:24,960 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 18:38:25,880 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:38:27,255 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 8 +2024-03-13 18:38:27,255 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 18:38:27,255 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:27,255 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-13 18:38:27,256 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:27,256 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-03-13 18:38:27,256 INFO SenderThread:877043 [job_builder.py:build():296] Attempting to build job artifact +2024-03-13 18:38:27,257 INFO SenderThread:877043 [job_builder.py:_get_source_type():426] is repo sourced job +2024-03-13 18:38:27,334 INFO SenderThread:877043 [job_builder.py:build():402] adding wandb-job metadata file +2024-03-13 18:38:27,362 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 9 +2024-03-13 18:38:27,363 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:27,363 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-13 18:38:27,368 DEBUG SenderThread:877043 [sender.py:send():382] send: artifact +2024-03-13 18:38:27,881 INFO Thread-12 :877043 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:38:27,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 18:38:28,449 INFO SenderThread:877043 [sender.py:send_artifact():1494] sent artifact job-https___github.com_TRI-ML_prismatic-vlms.git_scripts_pretrain.py - {'id': 'QXJ0aWZhY3Q6NzU0NjY3NjQ2', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE0MzcwMzk1OA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NzU0NjY3NjQ2', 'versionIndex': 9}}} +2024-03-13 18:38:28,449 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:28,449 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-03-13 18:38:28,449 INFO SenderThread:877043 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-13 18:38:28,882 INFO SenderThread:877043 [dir_watcher.py:finish():388] scan: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files +2024-03-13 18:38:28,883 INFO SenderThread:877043 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/config.yaml config.yaml +2024-03-13 18:38:28,883 INFO SenderThread:877043 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/conda-environment.yaml conda-environment.yaml +2024-03-13 18:38:28,883 INFO SenderThread:877043 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-metadata.json wandb-metadata.json +2024-03-13 18:38:28,883 INFO SenderThread:877043 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/requirements.txt requirements.txt +2024-03-13 18:38:28,883 INFO SenderThread:877043 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log output.log +2024-03-13 18:38:28,884 INFO SenderThread:877043 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json wandb-summary.json +2024-03-13 18:38:28,890 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 10 +2024-03-13 18:38:28,890 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 18:38:28,892 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:28,892 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-13 18:38:28,893 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:28,893 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-03-13 18:38:28,893 INFO SenderThread:877043 [file_pusher.py:finish():172] shutting down file pusher +2024-03-13 18:38:28,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 18:38:28,961 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 18:38:29,584 INFO wandb-upload_1:877043 [upload_job.py:push():131] Uploaded file runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/conda-environment.yaml +2024-03-13 18:38:29,700 INFO wandb-upload_4:877043 [upload_job.py:push():131] Uploaded file runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/wandb-summary.json +2024-03-13 18:38:29,701 INFO wandb-upload_0:877043 [upload_job.py:push():131] Uploaded file runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/config.yaml +2024-03-13 18:38:29,715 INFO wandb-upload_2:877043 [upload_job.py:push():131] Uploaded file runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/requirements.txt +2024-03-13 18:38:29,726 INFO wandb-upload_3:877043 [upload_job.py:push():131] Uploaded file runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/files/output.log +2024-03-13 18:38:29,927 INFO Thread-11 (_thread_body):877043 [sender.py:transition_state():617] send defer: 11 +2024-03-13 18:38:29,927 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:29,927 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-13 18:38:29,927 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:29,927 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-03-13 18:38:29,927 INFO SenderThread:877043 [file_pusher.py:join():178] waiting for file pusher +2024-03-13 18:38:29,927 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 12 +2024-03-13 18:38:29,928 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:29,928 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-13 18:38:29,928 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:29,928 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:29,928 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-03-13 18:38:29,928 INFO SenderThread:877043 [file_stream.py:finish():595] file stream finish called +2024-03-13 18:38:29,961 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 18:38:30,566 INFO SenderThread:877043 [file_stream.py:finish():599] file stream finish is done +2024-03-13 18:38:30,566 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 13 +2024-03-13 18:38:30,566 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 18:38:30,566 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:30,566 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-13 18:38:30,566 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:30,566 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-03-13 18:38:30,567 INFO SenderThread:877043 [sender.py:transition_state():617] send defer: 14 +2024-03-13 18:38:30,567 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: defer +2024-03-13 18:38:30,567 INFO HandlerThread:877043 [handler.py:handle_request_defer():172] handle defer: 14 +2024-03-13 18:38:30,567 DEBUG SenderThread:877043 [sender.py:send():382] send: final +2024-03-13 18:38:30,567 DEBUG SenderThread:877043 [sender.py:send():382] send: footer +2024-03-13 18:38:30,567 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: defer +2024-03-13 18:38:30,567 INFO SenderThread:877043 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-03-13 18:38:30,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 18:38:30,568 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 18:38:30,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:38:30,568 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: server_info +2024-03-13 18:38:30,568 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: server_info +2024-03-13 18:38:30,613 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: get_summary +2024-03-13 18:38:30,613 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: sampled_history +2024-03-13 18:38:30,614 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: job_info +2024-03-13 18:38:30,770 DEBUG SenderThread:877043 [sender.py:send_request():409] send_request: job_info +2024-03-13 18:38:30,771 DEBUG HandlerThread:877043 [handler.py:handle_request():146] handle_request: shutdown +2024-03-13 18:38:30,771 INFO HandlerThread:877043 [handler.py:finish():869] shutting down handler +2024-03-13 18:38:31,615 INFO WriterThread:877043 [datastore.py:close():296] close: runs/s1_s2_qformer2_128/wandb/run-20240313_112804-pairvt6d/run-pairvt6d.wandb +2024-03-13 18:38:31,771 INFO SenderThread:877043 [sender.py:finish():1572] shutting down sender +2024-03-13 18:38:31,771 INFO SenderThread:877043 [file_pusher.py:finish():172] shutting down file pusher +2024-03-13 18:38:31,771 INFO SenderThread:877043 [file_pusher.py:join():178] waiting for file pusher